From b1efa9f06b30931108b5ea45adfa8e518d07a182 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 16:41:56 +1300 Subject: [PATCH 01/10] Move tests in src/stream_safe.rs to tests/ Defining various glue intermediaries in __test_api --- src/__test_api.rs | 28 ++++++++++++++++++++ src/lib.rs | 3 ++- src/stream_safe.rs | 63 -------------------------------------------- tests/stream_safe.rs | 56 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 64 deletions(-) create mode 100644 src/__test_api.rs create mode 100644 tests/stream_safe.rs diff --git a/src/__test_api.rs b/src/__test_api.rs new file mode 100644 index 0000000..2c95c0c --- /dev/null +++ b/src/__test_api.rs @@ -0,0 +1,28 @@ +pub use crate::stream_safe::StreamSafe; +use crate::stream_safe; +use crate::lookups; +pub use crate::lookups::canonical_combining_class; +pub use crate::normalize::decompose_compatible; +pub use crate::normalization_tests::NORMALIZATION_TESTS; + +pub struct Decomposition { + pub leading_nonstarters: usize, + pub trailing_nonstarters: usize, + pub decomposition_len: usize, +} +impl Into for stream_safe::Decomposition { + fn into (self: stream_safe::Decomposition) -> Decomposition { + Decomposition { + leading_nonstarters: self.leading_nonstarters, + trailing_nonstarters: self.trailing_nonstarters, + decomposition_len: self.decomposition_len, + } + } +} + +pub fn classify_nonstarters(c: char) -> Decomposition { + stream_safe::classify_nonstarters(c).into() +} +pub fn stream_safe(s: &str) -> String { + StreamSafe::new(s.chars()).collect() +} diff --git a/src/lib.rs b/src/lib.rs index 7a4b4e0..63d3495 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -75,8 +75,9 @@ mod tables; #[cfg(test)] mod test; -#[cfg(test)] mod normalization_tests; +#[doc(hidden)] +pub mod __test_api; /// Methods for composing and decomposing characters. pub mod char { diff --git a/src/stream_safe.rs b/src/stream_safe.rs index 38bb42c..69123ca 100644 --- a/src/stream_safe.rs +++ b/src/stream_safe.rs @@ -103,66 +103,3 @@ pub(crate) fn classify_nonstarters(c: char) -> Decomposition { } } } - -#[cfg(test)] -mod tests { - use super::{ - StreamSafe, - classify_nonstarters, - }; - use std::char; - use normalization_tests::NORMALIZATION_TESTS; - use normalize::decompose_compatible; - use lookups::canonical_combining_class; - - fn stream_safe(s: &str) -> String { - StreamSafe::new(s.chars()).collect() - } - - #[test] - fn test_normalization_tests_unaffected() { - for test in NORMALIZATION_TESTS { - for &s in &[test.source, test.nfc, test.nfd, test.nfkc, test.nfkd] { - assert_eq!(stream_safe(s), s); - } - } - } - - #[test] - fn test_simple() { - let technically_okay = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}ngerzone"; - assert_eq!(stream_safe(technically_okay), technically_okay); - - let too_much = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}ngerzone"; - assert_ne!(stream_safe(too_much), too_much); - } - - #[test] - fn test_classify_nonstarters() { - // Highest character in the `compat_fully_decomp` table is 2FA1D - for ch in 0..0x2FA1E { - let ch = match char::from_u32(ch) { - Some(c) => c, - None => continue, - }; - let c = classify_nonstarters(ch); - let mut s = vec![]; - decompose_compatible(ch, |c| s.push(c)); - - assert_eq!(s.len(), c.decomposition_len); - - let num_leading = s - .iter() - .take_while(|&c| canonical_combining_class(*c) != 0) - .count(); - let num_trailing = s - .iter() - .rev() - .take_while(|&c| canonical_combining_class(*c) != 0) - .count(); - - assert_eq!(num_leading, c.leading_nonstarters); - assert_eq!(num_trailing, c.trailing_nonstarters); - } - } -} diff --git a/tests/stream_safe.rs b/tests/stream_safe.rs new file mode 100644 index 0000000..3e061ee --- /dev/null +++ b/tests/stream_safe.rs @@ -0,0 +1,56 @@ +extern crate unicode_normalization; +use unicode_normalization::__test_api::{ + classify_nonstarters, + canonical_combining_class, + decompose_compatible, + stream_safe, + NORMALIZATION_TESTS +}; +use std::char; + +#[test] +fn test_normalization_tests_unaffected() { + for test in NORMALIZATION_TESTS { + for &s in &[test.source, test.nfc, test.nfd, test.nfkc, test.nfkd] { + assert_eq!(stream_safe(s), s); + } + } +} + +#[test] +fn test_simple() { + let technically_okay = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}ngerzone"; + assert_eq!(stream_safe(technically_okay), technically_okay); + + let too_much = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}ngerzone"; + assert_ne!(stream_safe(too_much), too_much); +} + +#[test] +fn test_classify_nonstarters() { + // Highest character in the `compat_fully_decomp` table is 2FA1D + for ch in 0..0x2FA1E { + let ch = match char::from_u32(ch) { + Some(c) => c, + None => continue, + }; + let c = classify_nonstarters(ch); + let mut s = vec![]; + decompose_compatible(ch, |c| s.push(c)); + + assert_eq!(s.len(), c.decomposition_len); + + let num_leading = s + .iter() + .take_while(|&c| canonical_combining_class(*c) != 0) + .count(); + let num_trailing = s + .iter() + .rev() + .take_while(|&c| canonical_combining_class(*c) != 0) + .count(); + + assert_eq!(num_leading, c.leading_nonstarters); + assert_eq!(num_trailing, c.trailing_nonstarters); + } +} From d42fff26fa2b5aa7e50f632ebe2ecfc834128213 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 16:50:24 +1300 Subject: [PATCH 02/10] Move src/test.rs to tests/test.rs Adding various __test_api shims in the process --- src/__test_api.rs | 4 ++++ src/lib.rs | 2 -- {src => tests}/test.rs | 13 ++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) rename {src => tests}/test.rs (95%) diff --git a/src/__test_api.rs b/src/__test_api.rs index 2c95c0c..4b31a4d 100644 --- a/src/__test_api.rs +++ b/src/__test_api.rs @@ -26,3 +26,7 @@ pub fn classify_nonstarters(c: char) -> Decomposition { pub fn stream_safe(s: &str) -> String { StreamSafe::new(s.chars()).collect() } + +pub mod quick_check { + pub use crate::quick_check::*; +} diff --git a/src/lib.rs b/src/lib.rs index 63d3495..16fdebc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,8 +73,6 @@ mod quick_check; mod stream_safe; mod tables; -#[cfg(test)] -mod test; mod normalization_tests; #[doc(hidden)] pub mod __test_api; diff --git a/src/test.rs b/tests/test.rs similarity index 95% rename from src/test.rs rename to tests/test.rs index 4c7d2eb..3a96c53 100644 --- a/src/test.rs +++ b/tests/test.rs @@ -1,3 +1,4 @@ + // Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. @@ -9,9 +10,11 @@ // except according to those terms. +extern crate unicode_normalization; + use std::char; -use super::UnicodeNormalization; -use super::char::is_combining_mark; +use unicode_normalization::UnicodeNormalization; +use unicode_normalization::char::is_combining_mark; #[test] @@ -97,7 +100,7 @@ fn test_nfkc() { #[test] fn test_official() { - use normalization_tests::NORMALIZATION_TESTS; + use unicode_normalization::__test_api::NORMALIZATION_TESTS; macro_rules! normString { ($method: ident, $input: expr) => { $input.$method().collect::() } } @@ -161,8 +164,8 @@ fn test_official() { #[test] fn test_quick_check() { - use normalization_tests::NORMALIZATION_TESTS; - use quick_check; + use unicode_normalization::__test_api::NORMALIZATION_TESTS; + use unicode_normalization::__test_api::quick_check; for test in NORMALIZATION_TESTS { assert!(quick_check::is_nfc(test.nfc)); assert!(quick_check::is_nfd(test.nfd)); From feb20933e9b31ce0b1afdcca11584c6bc5bbea4b Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 16:56:01 +1300 Subject: [PATCH 03/10] Move compose_hangul test to own file --- src/__test_api.rs | 6 ++++++ src/normalize.rs | 15 +-------------- tests/normalize.rs | 11 +++++++++++ 3 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 tests/normalize.rs diff --git a/src/__test_api.rs b/src/__test_api.rs index 4b31a4d..6e87d76 100644 --- a/src/__test_api.rs +++ b/src/__test_api.rs @@ -1,6 +1,8 @@ pub use crate::stream_safe::StreamSafe; use crate::stream_safe; use crate::lookups; +use crate::normalize; + pub use crate::lookups::canonical_combining_class; pub use crate::normalize::decompose_compatible; pub use crate::normalization_tests::NORMALIZATION_TESTS; @@ -27,6 +29,10 @@ pub fn stream_safe(s: &str) -> String { StreamSafe::new(s.chars()).collect() } +pub fn compose_hangul(a: char, b: char) -> Option { + normalize::compose_hangul(a,b) +} + pub mod quick_check { pub use crate::quick_check::*; } diff --git a/src/normalize.rs b/src/normalize.rs index 87456df..7460b03 100644 --- a/src/normalize.rs +++ b/src/normalize.rs @@ -120,7 +120,7 @@ pub(crate) fn hangul_decomposition_length(s: char) -> usize { #[allow(unsafe_code)] #[inline(always)] #[allow(ellipsis_inclusive_range_patterns)] -fn compose_hangul(a: char, b: char) -> Option { +pub (crate) fn compose_hangul(a: char, b: char) -> Option { let (a, b) = (a as u32, b as u32); match (a, b) { // Compose a leading consonant and a vowel together into an LV_Syllable @@ -138,16 +138,3 @@ fn compose_hangul(a: char, b: char) -> Option { _ => None, } } - -#[cfg(test)] -mod tests { - use super::compose_hangul; - - // Regression test from a bugfix where we were composing an LV_Syllable with - // T_BASE directly. (We should only compose an LV_Syllable with a character - // in the range `T_BASE + 1 ... T_LAST`.) - #[test] - fn test_hangul_composition() { - assert_eq!(compose_hangul('\u{c8e0}', '\u{11a7}'), None); - } -} diff --git a/tests/normalize.rs b/tests/normalize.rs new file mode 100644 index 0000000..99833ce --- /dev/null +++ b/tests/normalize.rs @@ -0,0 +1,11 @@ +extern crate unicode_normalization; + +use unicode_normalization::__test_api::compose_hangul; + +// Regression test from a bugfix where we were composing an LV_Syllable with +// T_BASE directly. (We should only compose an LV_Syllable with a character +// in the range `T_BASE + 1 ... T_LAST`.) +#[test] +fn test_hangul_composition() { + assert_eq!(compose_hangul('\u{c8e0}', '\u{11a7}'), None); +} From c3a0033889429a4eacf3d17a982db8ea14333d21 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 17:01:02 +1300 Subject: [PATCH 04/10] Move quick_check tests to tests/ --- src/quick_check.rs | 27 --------------------------- tests/quickcheck.rs | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 27 deletions(-) create mode 100644 tests/quickcheck.rs diff --git a/src/quick_check.rs b/src/quick_check.rs index 49b1460..96d1fe8 100644 --- a/src/quick_check.rs +++ b/src/quick_check.rs @@ -161,30 +161,3 @@ pub fn is_nfd_stream_safe(s: &str) -> bool { IsNormalized::Maybe => s.chars().eq(s.chars().stream_safe().nfd()), } } - -#[cfg(test)] -mod tests { - use super::{ - IsNormalized, - is_nfc_stream_safe_quick, - is_nfd_stream_safe_quick, - }; - - #[test] - fn test_stream_safe_nfd() { - let okay = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone"; - assert_eq!(is_nfd_stream_safe_quick(okay.chars()), IsNormalized::Yes); - - let too_much = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone"; - assert_eq!(is_nfd_stream_safe_quick(too_much.chars()), IsNormalized::No); - } - - #[test] - fn test_stream_safe_nfc() { - let okay = "ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y"; - assert_eq!(is_nfc_stream_safe_quick(okay.chars()), IsNormalized::Maybe); - - let too_much = "not ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y"; - assert_eq!(is_nfc_stream_safe_quick(too_much.chars()), IsNormalized::No); - } -} diff --git a/tests/quickcheck.rs b/tests/quickcheck.rs new file mode 100644 index 0000000..11e1091 --- /dev/null +++ b/tests/quickcheck.rs @@ -0,0 +1,25 @@ +extern crate unicode_normalization; + +use unicode_normalization::{ + IsNormalized, + is_nfc_stream_safe_quick, + is_nfd_stream_safe_quick, +}; + +#[test] +fn test_stream_safe_nfd() { + let okay = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone"; + assert_eq!(is_nfd_stream_safe_quick(okay.chars()), IsNormalized::Yes); + + let too_much = "Da\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}ngerzone"; + assert_eq!(is_nfd_stream_safe_quick(too_much.chars()), IsNormalized::No); +} + +#[test] +fn test_stream_safe_nfc() { + let okay = "ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y"; + assert_eq!(is_nfc_stream_safe_quick(okay.chars()), IsNormalized::Maybe); + + let too_much = "not ok\u{e0}\u{031b}\u{0316}\u{0317}\u{0318}\u{0319}\u{031c}\u{031d}\u{031e}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{031a}y"; + assert_eq!(is_nfc_stream_safe_quick(too_much.chars()), IsNormalized::No); +} From 6cf214e5253a1376b5fb7c2246e73c5bfec76513 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 17:07:12 +1300 Subject: [PATCH 05/10] Move normalization_tests.rs to tests/ --- src/__test_api.rs | 7 ++++++- src/lib.rs | 1 - {src => tests}/normalization_tests.rs | 0 tests/stream_safe.rs | 3 ++- tests/test.rs | 5 ++--- 5 files changed, 10 insertions(+), 6 deletions(-) rename {src => tests}/normalization_tests.rs (100%) diff --git a/src/__test_api.rs b/src/__test_api.rs index 6e87d76..780166b 100644 --- a/src/__test_api.rs +++ b/src/__test_api.rs @@ -1,3 +1,9 @@ +// This crate comprises hacks and glue required to test private functions from tests/ +// +// Keep this as slim as possible. +// +// If you're caught using this outside this crates tests/, you get to clean up the mess. + pub use crate::stream_safe::StreamSafe; use crate::stream_safe; use crate::lookups; @@ -5,7 +11,6 @@ use crate::normalize; pub use crate::lookups::canonical_combining_class; pub use crate::normalize::decompose_compatible; -pub use crate::normalization_tests::NORMALIZATION_TESTS; pub struct Decomposition { pub leading_nonstarters: usize, diff --git a/src/lib.rs b/src/lib.rs index 16fdebc..67fbe53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,7 +73,6 @@ mod quick_check; mod stream_safe; mod tables; -mod normalization_tests; #[doc(hidden)] pub mod __test_api; diff --git a/src/normalization_tests.rs b/tests/normalization_tests.rs similarity index 100% rename from src/normalization_tests.rs rename to tests/normalization_tests.rs diff --git a/tests/stream_safe.rs b/tests/stream_safe.rs index 3e061ee..bb8417a 100644 --- a/tests/stream_safe.rs +++ b/tests/stream_safe.rs @@ -4,8 +4,9 @@ use unicode_normalization::__test_api::{ canonical_combining_class, decompose_compatible, stream_safe, - NORMALIZATION_TESTS }; +mod normalization_tests; +use normalization_tests::NORMALIZATION_TESTS; use std::char; #[test] diff --git a/tests/test.rs b/tests/test.rs index 3a96c53..5112716 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -15,7 +15,8 @@ extern crate unicode_normalization; use std::char; use unicode_normalization::UnicodeNormalization; use unicode_normalization::char::is_combining_mark; - +mod normalization_tests; +use normalization_tests::NORMALIZATION_TESTS; #[test] fn test_nfd() { @@ -100,7 +101,6 @@ fn test_nfkc() { #[test] fn test_official() { - use unicode_normalization::__test_api::NORMALIZATION_TESTS; macro_rules! normString { ($method: ident, $input: expr) => { $input.$method().collect::() } } @@ -164,7 +164,6 @@ fn test_official() { #[test] fn test_quick_check() { - use unicode_normalization::__test_api::NORMALIZATION_TESTS; use unicode_normalization::__test_api::quick_check; for test in NORMALIZATION_TESTS { assert!(quick_check::is_nfc(test.nfc)); From 553cd4ccc0cf4dd4139f4a038508f49792f96389 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 17:11:55 +1300 Subject: [PATCH 06/10] Factor stream-safe tests dependent on NORMALIZATION_TESTS to own file --- tests/stream_safe.rs | 11 ----------- tests/stream_safe_normalization.rs | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 tests/stream_safe_normalization.rs diff --git a/tests/stream_safe.rs b/tests/stream_safe.rs index bb8417a..b6cbfe3 100644 --- a/tests/stream_safe.rs +++ b/tests/stream_safe.rs @@ -5,19 +5,8 @@ use unicode_normalization::__test_api::{ decompose_compatible, stream_safe, }; -mod normalization_tests; -use normalization_tests::NORMALIZATION_TESTS; use std::char; -#[test] -fn test_normalization_tests_unaffected() { - for test in NORMALIZATION_TESTS { - for &s in &[test.source, test.nfc, test.nfd, test.nfkc, test.nfkd] { - assert_eq!(stream_safe(s), s); - } - } -} - #[test] fn test_simple() { let technically_okay = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}ngerzone"; diff --git a/tests/stream_safe_normalization.rs b/tests/stream_safe_normalization.rs new file mode 100644 index 0000000..55ba960 --- /dev/null +++ b/tests/stream_safe_normalization.rs @@ -0,0 +1,17 @@ +extern crate unicode_normalization; +use unicode_normalization::__test_api::{ + stream_safe, +}; + +mod normalization_tests; +use normalization_tests::NORMALIZATION_TESTS; + +#[test] +fn test_normalization_tests_unaffected() { + for test in NORMALIZATION_TESTS { + for &s in &[test.source, test.nfc, test.nfd, test.nfkc, test.nfkd] { + assert_eq!(stream_safe(s), s); + } + } +} + From e7c5602f43a1c10e0b0eee5fae8aae305918eda8 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 17:18:49 +1300 Subject: [PATCH 07/10] Factor normalization data requiring tests into own file --- tests/test.rs | 87 ----------------------------------- tests/test_normalization.rs | 91 +++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 87 deletions(-) create mode 100644 tests/test_normalization.rs diff --git a/tests/test.rs b/tests/test.rs index 5112716..d9a6cdb 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -15,8 +15,6 @@ extern crate unicode_normalization; use std::char; use unicode_normalization::UnicodeNormalization; use unicode_normalization::char::is_combining_mark; -mod normalization_tests; -use normalization_tests::NORMALIZATION_TESTS; #[test] fn test_nfd() { @@ -99,91 +97,6 @@ fn test_nfkc() { t!("a\u{300}\u{305}\u{315}\u{5ae}b", "\u{e0}\u{5ae}\u{305}\u{315}b"); } -#[test] -fn test_official() { - macro_rules! normString { - ($method: ident, $input: expr) => { $input.$method().collect::() } - } - - for test in NORMALIZATION_TESTS { - // these invariants come from the CONFORMANCE section of - // http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt - { - let r1 = normString!(nfc, test.source); - let r2 = normString!(nfc, test.nfc); - let r3 = normString!(nfc, test.nfd); - let r4 = normString!(nfc, test.nfkc); - let r5 = normString!(nfc, test.nfkd); - assert_eq!(test.nfc, &r1[..]); - assert_eq!(test.nfc, &r2[..]); - assert_eq!(test.nfc, &r3[..]); - assert_eq!(test.nfkc, &r4[..]); - assert_eq!(test.nfkc, &r5[..]); - } - - { - let r1 = normString!(nfd, test.source); - let r2 = normString!(nfd, test.nfc); - let r3 = normString!(nfd, test.nfd); - let r4 = normString!(nfd, test.nfkc); - let r5 = normString!(nfd, test.nfkd); - assert_eq!(test.nfd, &r1[..]); - assert_eq!(test.nfd, &r2[..]); - assert_eq!(test.nfd, &r3[..]); - assert_eq!(test.nfkd, &r4[..]); - assert_eq!(test.nfkd, &r5[..]); - } - - { - let r1 = normString!(nfkc, test.source); - let r2 = normString!(nfkc, test.nfc); - let r3 = normString!(nfkc, test.nfd); - let r4 = normString!(nfkc, test.nfkc); - let r5 = normString!(nfkc, test.nfkd); - assert_eq!(test.nfkc, &r1[..]); - assert_eq!(test.nfkc, &r2[..]); - assert_eq!(test.nfkc, &r3[..]); - assert_eq!(test.nfkc, &r4[..]); - assert_eq!(test.nfkc, &r5[..]); - } - - { - let r1 = normString!(nfkd, test.source); - let r2 = normString!(nfkd, test.nfc); - let r3 = normString!(nfkd, test.nfd); - let r4 = normString!(nfkd, test.nfkc); - let r5 = normString!(nfkd, test.nfkd); - assert_eq!(test.nfkd, &r1[..]); - assert_eq!(test.nfkd, &r2[..]); - assert_eq!(test.nfkd, &r3[..]); - assert_eq!(test.nfkd, &r4[..]); - assert_eq!(test.nfkd, &r5[..]); - } - } -} - -#[test] -fn test_quick_check() { - use unicode_normalization::__test_api::quick_check; - for test in NORMALIZATION_TESTS { - assert!(quick_check::is_nfc(test.nfc)); - assert!(quick_check::is_nfd(test.nfd)); - assert!(quick_check::is_nfkc(test.nfkc)); - assert!(quick_check::is_nfkd(test.nfkd)); - if test.nfc != test.nfd { - assert!(!quick_check::is_nfc(test.nfd)); - assert!(!quick_check::is_nfd(test.nfc)); - } - if test.nfkc != test.nfc { - assert!(!quick_check::is_nfkc(test.nfc)); - assert!(quick_check::is_nfc(test.nfkc)); - } - if test.nfkd != test.nfd { - assert!(!quick_check::is_nfkd(test.nfd)); - assert!(quick_check::is_nfd(test.nfkd)); - } - } -} #[test] fn test_is_combining_mark_ascii() { diff --git a/tests/test_normalization.rs b/tests/test_normalization.rs new file mode 100644 index 0000000..356d07e --- /dev/null +++ b/tests/test_normalization.rs @@ -0,0 +1,91 @@ +extern crate unicode_normalization; +use unicode_normalization::UnicodeNormalization; +mod normalization_tests; +use normalization_tests::NORMALIZATION_TESTS; + +#[test] +fn test_official() { + macro_rules! normString { + ($method: ident, $input: expr) => { $input.$method().collect::() } + } + + for test in NORMALIZATION_TESTS { + // these invariants come from the CONFORMANCE section of + // http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt + { + let r1 = normString!(nfc, test.source); + let r2 = normString!(nfc, test.nfc); + let r3 = normString!(nfc, test.nfd); + let r4 = normString!(nfc, test.nfkc); + let r5 = normString!(nfc, test.nfkd); + assert_eq!(test.nfc, &r1[..]); + assert_eq!(test.nfc, &r2[..]); + assert_eq!(test.nfc, &r3[..]); + assert_eq!(test.nfkc, &r4[..]); + assert_eq!(test.nfkc, &r5[..]); + } + + { + let r1 = normString!(nfd, test.source); + let r2 = normString!(nfd, test.nfc); + let r3 = normString!(nfd, test.nfd); + let r4 = normString!(nfd, test.nfkc); + let r5 = normString!(nfd, test.nfkd); + assert_eq!(test.nfd, &r1[..]); + assert_eq!(test.nfd, &r2[..]); + assert_eq!(test.nfd, &r3[..]); + assert_eq!(test.nfkd, &r4[..]); + assert_eq!(test.nfkd, &r5[..]); + } + + { + let r1 = normString!(nfkc, test.source); + let r2 = normString!(nfkc, test.nfc); + let r3 = normString!(nfkc, test.nfd); + let r4 = normString!(nfkc, test.nfkc); + let r5 = normString!(nfkc, test.nfkd); + assert_eq!(test.nfkc, &r1[..]); + assert_eq!(test.nfkc, &r2[..]); + assert_eq!(test.nfkc, &r3[..]); + assert_eq!(test.nfkc, &r4[..]); + assert_eq!(test.nfkc, &r5[..]); + } + + { + let r1 = normString!(nfkd, test.source); + let r2 = normString!(nfkd, test.nfc); + let r3 = normString!(nfkd, test.nfd); + let r4 = normString!(nfkd, test.nfkc); + let r5 = normString!(nfkd, test.nfkd); + assert_eq!(test.nfkd, &r1[..]); + assert_eq!(test.nfkd, &r2[..]); + assert_eq!(test.nfkd, &r3[..]); + assert_eq!(test.nfkd, &r4[..]); + assert_eq!(test.nfkd, &r5[..]); + } + } +} + +#[test] +fn test_quick_check() { + use unicode_normalization::__test_api::quick_check; + for test in NORMALIZATION_TESTS { + assert!(quick_check::is_nfc(test.nfc)); + assert!(quick_check::is_nfd(test.nfd)); + assert!(quick_check::is_nfkc(test.nfkc)); + assert!(quick_check::is_nfkd(test.nfkd)); + if test.nfc != test.nfd { + assert!(!quick_check::is_nfc(test.nfd)); + assert!(!quick_check::is_nfd(test.nfc)); + } + if test.nfkc != test.nfc { + assert!(!quick_check::is_nfkc(test.nfc)); + assert!(quick_check::is_nfc(test.nfkc)); + } + if test.nfkd != test.nfd { + assert!(!quick_check::is_nfkd(test.nfd)); + assert!(quick_check::is_nfd(test.nfkd)); + } + } +} + From f55b99cbe3c80ecfad614dfa478a1acab9e204c4 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 17:20:22 +1300 Subject: [PATCH 08/10] Change excluded files This excludes: - test/normalization_tests.rs And 2 tests that consume it: - test/test_normalization.rs - test/stream_safe_normalization.rs Tests should pass when this crate is published without these files, albeit resulting in slightly lower test quality. --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e4727c7..f7c78ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ Decomposition and Recomposition, as described in Unicode Standard Annex #15. """ -exclude = [ "target/*", "Cargo.lock", "scripts/tmp", "*.txt", "src/normalization_tests.rs", "src/test.rs" ] +exclude = [ "target/*", "Cargo.lock", "scripts/tmp", "*.txt", "tests/normalization_tests.rs", "tests/stream_safe_normalization.rs", "tests/test_normalization.rs" ] [dependencies] -smallvec = "0.6" \ No newline at end of file +smallvec = "0.6" From aa43763500eab6718d222df858e512a98adb8da0 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Thu, 7 Nov 2019 17:22:34 +1300 Subject: [PATCH 09/10] Remove unused import in __test_api --- src/__test_api.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/__test_api.rs b/src/__test_api.rs index 780166b..e70d808 100644 --- a/src/__test_api.rs +++ b/src/__test_api.rs @@ -6,7 +6,6 @@ pub use crate::stream_safe::StreamSafe; use crate::stream_safe; -use crate::lookups; use crate::normalize; pub use crate::lookups::canonical_combining_class; From 226dcb85b391698e6815c7c19256e8e86bf69216 Mon Sep 17 00:00:00 2001 From: Kent Fredric Date: Tue, 29 Oct 2019 15:00:18 +1300 Subject: [PATCH 10/10] Add mechanisms for testing published crates for coherence. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8f11202..7bbfda1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ sudo: false script: - cargo build --verbose - cargo test --verbose + - cargo package + - cd target/package/unicode-normalization-* + - cargo test --verbose notifications: email: on_success: never