#![forbid(unsafe_code)]
#![cfg_attr(feature="flame_it", feature(plugin, custom_attribute))]
#![cfg_attr(feature="flame_it", plugin(flamer))]
#[macro_use]
extern crate matches;
#[cfg(feature = "serde")]
#[macro_use]
extern crate serde;
#[cfg(all(feature = "serde", test))]
extern crate serde_test;
#[cfg(feature = "flame_it")]
extern crate flame;
pub mod deprecated;
pub mod format_chars;
pub mod level;
mod char_data;
mod explicit;
mod implicit;
mod prepare;
pub use char_data::{BidiClass, bidi_class, UNICODE_VERSION};
pub use level::{Level, LTR_LEVEL, RTL_LEVEL};
pub use prepare::LevelRun;
use std::borrow::Cow;
use std::cmp::{max, min};
use std::iter::repeat;
use std::ops::Range;
use BidiClass::*;
use format_chars as chars;
#[derive(Debug, PartialEq)]
pub struct ParagraphInfo {
pub range: Range<usize>,
pub level: Level,
}
#[derive(PartialEq, Debug)]
pub struct InitialInfo<'text> {
pub text: &'text str,
pub original_classes: Vec<BidiClass>,
pub paragraphs: Vec<ParagraphInfo>,
}
impl<'text> InitialInfo<'text> {
#[cfg_attr(feature = "flame_it", flame)]
pub fn new(text: &str, default_para_level: Option<Level>) -> InitialInfo {
let mut original_classes = Vec::with_capacity(text.len());
let mut isolate_stack = Vec::new();
let mut paragraphs = Vec::new();
let mut para_start = 0;
let mut para_level = default_para_level;
#[cfg(feature = "flame_it")] flame::start("InitialInfo::new(): iter text.char_indices()");
for (i, c) in text.char_indices() {
let class = bidi_class(c);
#[cfg(feature = "flame_it")] flame::start("original_classes.extend()");
original_classes.extend(repeat(class).take(c.len_utf8()));
#[cfg(feature = "flame_it")] flame::end("original_classes.extend()");
match class {
B => {
let para_end = i + c.len_utf8();
paragraphs.push(ParagraphInfo {
range: para_start..para_end,
level: para_level.unwrap_or(LTR_LEVEL),
});
para_start = para_end;
para_level = default_para_level;
isolate_stack.clear();
}
L | R | AL => {
match isolate_stack.last() {
Some(&start) => {
if original_classes[start] == FSI {
for j in 0..chars::FSI.len_utf8() {
original_classes[start + j] =
if class == L { LRI } else { RLI };
}
}
}
None => {
if para_level.is_none() {
para_level = Some(if class != L { RTL_LEVEL } else { LTR_LEVEL });
}
}
}
}
RLI | LRI | FSI => {
isolate_stack.push(i);
}
PDI => {
isolate_stack.pop();
}
_ => {}
}
}
if para_start < text.len() {
paragraphs.push(ParagraphInfo {
range: para_start..text.len(),
level: para_level.unwrap_or(LTR_LEVEL),
});
}
assert_eq!(original_classes.len(), text.len());
#[cfg(feature = "flame_it")] flame::end("InitialInfo::new(): iter text.char_indices()");
InitialInfo {
text,
original_classes,
paragraphs,
}
}
}
#[derive(Debug, PartialEq)]
pub struct BidiInfo<'text> {
pub text: &'text str,
pub original_classes: Vec<BidiClass>,
pub levels: Vec<Level>,
pub paragraphs: Vec<ParagraphInfo>,
}
impl<'text> BidiInfo<'text> {
#[cfg_attr(feature = "flame_it", flame)]
pub fn new(text: &str, default_para_level: Option<Level>) -> BidiInfo {
let InitialInfo {
original_classes,
paragraphs,
..
} = InitialInfo::new(text, default_para_level);
let mut levels = Vec::<Level>::with_capacity(text.len());
let mut processing_classes = original_classes.clone();
for para in ¶graphs {
let text = &text[para.range.clone()];
let original_classes = &original_classes[para.range.clone()];
let processing_classes = &mut processing_classes[para.range.clone()];
let new_len = levels.len() + para.range.len();
levels.resize(new_len, para.level);
let levels = &mut levels[para.range.clone()];
explicit::compute(
text,
para.level,
original_classes,
levels,
processing_classes,
);
let sequences = prepare::isolating_run_sequences(para.level, original_classes, levels);
for sequence in &sequences {
implicit::resolve_weak(sequence, processing_classes);
implicit::resolve_neutral(sequence, levels, processing_classes);
}
implicit::resolve_levels(processing_classes, levels);
assign_levels_to_removed_chars(para.level, original_classes, levels);
}
BidiInfo {
text,
original_classes,
paragraphs,
levels,
}
}
#[cfg_attr(feature = "flame_it", flame)]
pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range<usize>) -> Vec<Level> {
let (levels, _) = self.visual_runs(para, line.clone());
levels
}
#[cfg_attr(feature = "flame_it", flame)]
pub fn reordered_levels_per_char(
&self,
para: &ParagraphInfo,
line: Range<usize>,
) -> Vec<Level> {
let levels = self.reordered_levels(para, line);
self.text.char_indices().map(|(i, _)| levels[i]).collect()
}
#[cfg_attr(feature = "flame_it", flame)]
pub fn reorder_line(&self, para: &ParagraphInfo, line: Range<usize>) -> Cow<'text, str> {
let (levels, runs) = self.visual_runs(para, line.clone());
if runs.iter().all(|run| levels[run.start].is_ltr()) {
return self.text[line.clone()].into();
}
let mut result = String::with_capacity(line.len());
for run in runs {
if levels[run.start].is_rtl() {
result.extend(self.text[run].chars().rev());
} else {
result.push_str(&self.text[run]);
}
}
result.into()
}
#[cfg_attr(feature = "flame_it", flame)]
pub fn visual_runs(
&self,
para: &ParagraphInfo,
line: Range<usize>,
) -> (Vec<Level>, Vec<LevelRun>) {
assert!(line.start <= self.levels.len());
assert!(line.end <= self.levels.len());
let mut levels = self.levels.clone();
let line_str: &str = &self.text[line.clone()];
let mut reset_from: Option<usize> = Some(0);
let mut reset_to: Option<usize> = None;
for (i, c) in line_str.char_indices() {
match self.original_classes[i] {
RLE | LRE | RLO | LRO | PDF | BN => {}
B | S => {
assert_eq!(reset_to, None);
reset_to = Some(i + c.len_utf8());
if reset_from == None {
reset_from = Some(i);
}
}
WS | FSI | LRI | RLI | PDI => {
if reset_from == None {
reset_from = Some(i);
}
}
_ => {
reset_from = None;
}
}
if let (Some(from), Some(to)) = (reset_from, reset_to) {
#[cfg_attr(feature = "cargo-clippy", allow(needless_range_loop))]
for j in from..to {
levels[j] = para.level;
}
reset_from = None;
reset_to = None;
}
}
if let Some(from) = reset_from {
#[cfg_attr(feature = "cargo-clippy", allow(needless_range_loop))]
for j in from..line_str.len() {
levels[j] = para.level;
}
}
let mut runs = Vec::new();
let mut start = line.start;
let mut run_level = levels[start];
let mut min_level = run_level;
let mut max_level = run_level;
for (i, &new_level) in levels.iter().enumerate().take(line.end).skip(start + 1) {
if new_level != run_level {
runs.push(start..i);
start = i;
run_level = new_level;
min_level = min(run_level, min_level);
max_level = max(run_level, max_level);
}
}
runs.push(start..line.end);
let run_count = runs.len();
min_level = min_level.new_lowest_ge_rtl().expect("Level error");
while max_level >= min_level {
let mut seq_start = 0;
while seq_start < run_count {
if self.levels[runs[seq_start].start] < max_level {
seq_start += 1;
continue;
}
let mut seq_end = seq_start + 1;
while seq_end < run_count {
if self.levels[runs[seq_end].start] < max_level {
break;
}
seq_end += 1;
}
runs[seq_start..seq_end].reverse();
seq_start = seq_end;
}
max_level.lower(1).expect(
"Lowering embedding level below zero",
);
}
(levels, runs)
}
#[inline]
pub fn has_rtl(&self) -> bool {
level::has_rtl(&self.levels)
}
}
#[cfg_attr(feature = "flame_it", flame)]
fn assign_levels_to_removed_chars(para_level: Level, classes: &[BidiClass], levels: &mut [Level]) {
for i in 0..levels.len() {
if prepare::removed_by_x9(classes[i]) {
levels[i] = if i > 0 { levels[i - 1] } else { para_level };
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_initial_text_info() {
let text = "a1";
assert_eq!(
InitialInfo::new(text, None),
InitialInfo {
text,
original_classes: vec![L, EN],
paragraphs: vec![
ParagraphInfo {
range: 0..2,
level: LTR_LEVEL,
},
],
}
);
let text = "غ א";
assert_eq!(
InitialInfo::new(text, None),
InitialInfo {
text,
original_classes: vec![AL, AL, WS, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..5,
level: RTL_LEVEL,
},
],
}
);
let text = "a\u{2029}b";
assert_eq!(
InitialInfo::new(text, None),
InitialInfo {
text,
original_classes: vec![L, B, B, B, L],
paragraphs: vec![
ParagraphInfo {
range: 0..4,
level: LTR_LEVEL,
},
ParagraphInfo {
range: 4..5,
level: LTR_LEVEL,
},
],
}
);
let text = format!("{}א{}a", chars::FSI, chars::PDI);
assert_eq!(
InitialInfo::new(&text, None),
InitialInfo {
text: &text,
original_classes: vec![RLI, RLI, RLI, R, R, PDI, PDI, PDI, L],
paragraphs: vec![
ParagraphInfo {
range: 0..9,
level: LTR_LEVEL,
},
],
}
);
}
#[test]
fn test_process_text() {
let text = "abc123";
assert_eq!(
BidiInfo::new(text, Some(LTR_LEVEL)),
BidiInfo {
text,
levels: Level::vec(&[0, 0, 0, 0, 0, 0]),
original_classes: vec![L, L, L, EN, EN, EN],
paragraphs: vec![
ParagraphInfo {
range: 0..6,
level: LTR_LEVEL,
},
],
}
);
let text = "abc אבג";
assert_eq!(
BidiInfo::new(text, Some(LTR_LEVEL)),
BidiInfo {
text,
levels: Level::vec(&[0, 0, 0, 0, 1, 1, 1, 1, 1, 1]),
original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
level: LTR_LEVEL,
},
],
}
);
assert_eq!(
BidiInfo::new(text, Some(RTL_LEVEL)),
BidiInfo {
text,
levels: Level::vec(&[2, 2, 2, 1, 1, 1, 1, 1, 1, 1]),
original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
level: RTL_LEVEL,
},
],
}
);
let text = "אבג abc";
assert_eq!(
BidiInfo::new(text, Some(LTR_LEVEL)),
BidiInfo {
text,
levels: Level::vec(&[1, 1, 1, 1, 1, 1, 0, 0, 0, 0]),
original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
level: LTR_LEVEL,
},
],
}
);
assert_eq!(
BidiInfo::new(text, None),
BidiInfo {
text,
levels: Level::vec(&[1, 1, 1, 1, 1, 1, 1, 2, 2, 2]),
original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
level: RTL_LEVEL,
},
],
}
);
let text = "غ2ظ א2ג";
assert_eq!(
BidiInfo::new(text, Some(LTR_LEVEL)),
BidiInfo {
text,
levels: Level::vec(&[1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]),
original_classes: vec![AL, AL, EN, AL, AL, WS, R, R, EN, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..11,
level: LTR_LEVEL,
},
],
}
);
let text = "a א.\nג";
assert_eq!(
BidiInfo::new(text, None),
BidiInfo {
text,
original_classes: vec![L, WS, R, R, CS, B, R, R],
levels: Level::vec(&[0, 0, 1, 1, 0, 0, 1, 1]),
paragraphs: vec![
ParagraphInfo {
range: 0..6,
level: LTR_LEVEL,
},
ParagraphInfo {
range: 6..8,
level: RTL_LEVEL,
},
],
}
);
let bidi_info = BidiInfo::new("\u{060B}\u{20CF}\u{06F9}", None);
assert_eq!(bidi_info.original_classes, vec![AL, AL, ET, ET, ET, EN, EN]);
}
#[test]
fn test_bidi_info_has_rtl() {
assert_eq!(BidiInfo::new("123", None).has_rtl(), false);
assert_eq!(BidiInfo::new("123", Some(LTR_LEVEL)).has_rtl(), false);
assert_eq!(BidiInfo::new("123", Some(RTL_LEVEL)).has_rtl(), false);
assert_eq!(BidiInfo::new("abc", None).has_rtl(), false);
assert_eq!(BidiInfo::new("abc", Some(LTR_LEVEL)).has_rtl(), false);
assert_eq!(BidiInfo::new("abc", Some(RTL_LEVEL)).has_rtl(), false);
assert_eq!(BidiInfo::new("abc 123", None).has_rtl(), false);
assert_eq!(BidiInfo::new("abc\n123", None).has_rtl(), false);
assert_eq!(BidiInfo::new("אבּג", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג", Some(LTR_LEVEL)).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג", Some(RTL_LEVEL)).has_rtl(), true);
assert_eq!(BidiInfo::new("abc אבּג", None).has_rtl(), true);
assert_eq!(BidiInfo::new("abc\nאבּג", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג abc", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג\nabc", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג 123", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג\n123", None).has_rtl(), true);
}
fn reorder_paras(text: &str) -> Vec<Cow<str>> {
let bidi_info = BidiInfo::new(text, None);
bidi_info
.paragraphs
.iter()
.map(|para| bidi_info.reorder_line(para, para.range.clone()))
.collect()
}
#[test]
fn test_reorder_line() {
assert_eq!(
reorder_paras("abc\ndef\nghi"),
vec!["abc\n", "def\n", "ghi"]
);
assert_eq!(
reorder_paras("ab1\nde2\ngh3"),
vec!["ab1\n", "de2\n", "gh3"]
);
assert_eq!(reorder_paras("abc\nابج"), vec!["abc\n", "جبا"]);
assert_eq!(reorder_paras("ابج\nabc"), vec!["\nجبا", "abc"]);
assert_eq!(reorder_paras("1.-2"), vec!["1.-2"]);
assert_eq!(reorder_paras("1-.2"), vec!["1-.2"]);
assert_eq!(reorder_paras("abc אבג"), vec!["abc גבא"]);
assert_eq!(reorder_paras("123 אבג"), vec!["גבא 123"]);
assert_eq!(reorder_paras("abc\u{202A}def"), vec!["abc\u{202A}def"]);
assert_eq!(
reorder_paras("abc\u{202A}def\u{202C}ghi"),
vec!["abc\u{202A}def\u{202C}ghi"]
);
assert_eq!(
reorder_paras("abc\u{2066}def\u{2069}ghi"),
vec!["abc\u{2066}def\u{2069}ghi"]
);
assert_eq!(
reorder_paras("\u{202B}abc אבג\u{202C}"),
vec!["\u{202B}\u{202C}גבא abc"]
);
assert_eq!(reorder_paras("אבג? אבג"), vec!["גבא ?גבא"]);
assert_eq!(reorder_paras("A אבג?"), vec!["A גבא?"]);
assert_eq!(
reorder_paras("A אבג?\u{200F}"),
vec!["A \u{200F}?גבא"]
);
assert_eq!(reorder_paras("אבג abc"), vec!["abc גבא"]);
assert_eq!(
reorder_paras("abc\u{2067}.-\u{2069}ghi"),
vec!["abc\u{2067}-.\u{2069}ghi"]
);
assert_eq!(
reorder_paras("Hello, \u{2068}\u{202E}world\u{202C}\u{2069}!"),
vec!["Hello, \u{2068}\u{202E}\u{202C}dlrow\u{2069}!"]
);
assert_eq!(reorder_paras("א(ב)ג."), vec![".ג)ב(א"]);
assert_eq!(
reorder_paras("אב(גד[&ef].)gh"),
vec!["ef].)gh&[דג(בא"]
);
}
fn reordered_levels_for_paras(text: &str) -> Vec<Vec<Level>> {
let bidi_info = BidiInfo::new(text, None);
bidi_info
.paragraphs
.iter()
.map(|para| bidi_info.reordered_levels(para, para.range.clone()))
.collect()
}
fn reordered_levels_per_char_for_paras(text: &str) -> Vec<Vec<Level>> {
let bidi_info = BidiInfo::new(text, None);
bidi_info
.paragraphs
.iter()
.map(|para| {
bidi_info.reordered_levels_per_char(para, para.range.clone())
})
.collect()
}
#[test]
fn test_reordered_levels() {
let text = "\u{2067}\u{2069}";
assert_eq!(
reordered_levels_for_paras(text),
vec![Level::vec(&[0, 0, 0, 0, 0, 0])]
);
assert_eq!(
reordered_levels_per_char_for_paras(text),
vec![Level::vec(&[0, 0])]
);
}
}
#[cfg(all(feature = "serde", test))]
mod serde_tests {
use serde_test::{Token, assert_tokens};
use super::*;
#[test]
fn test_levels() {
let text = "abc אבג";
let bidi_info = BidiInfo::new(text, None);
let levels = bidi_info.levels;
assert_eq!(text.as_bytes().len(), 10);
assert_eq!(levels.len(), 10);
assert_tokens(
&levels,
&[
Token::Seq { len: Some(10) },
Token::NewtypeStruct { name: "Level" },
Token::U8(0),
Token::NewtypeStruct { name: "Level" },
Token::U8(0),
Token::NewtypeStruct { name: "Level" },
Token::U8(0),
Token::NewtypeStruct { name: "Level" },
Token::U8(0),
Token::NewtypeStruct { name: "Level" },
Token::U8(1),
Token::NewtypeStruct { name: "Level" },
Token::U8(1),
Token::NewtypeStruct { name: "Level" },
Token::U8(1),
Token::NewtypeStruct { name: "Level" },
Token::U8(1),
Token::NewtypeStruct { name: "Level" },
Token::U8(1),
Token::NewtypeStruct { name: "Level" },
Token::U8(1),
Token::SeqEnd,
],
);
}
}