Tokenizer-lib

Rust中构建解析器的标记化工具

示例

缓冲标记通道

use tokenizer_lib::{BufferedTokenQueue, Token, TokenReader, TokenSender, TokenTrait};

#[derive(PartialEq, Debug)]
struct Span(pub u32, pub u32);

#[derive(PartialEq, Debug)]
struct N(pub u32);

impl TokenTrait for N {}

let mut btq = BufferedTokenQueue::new();
btq.push(Token(N(12), Span(0, 2)));
btq.push(Token(N(32), Span(2, 4)));
btq.push(Token(N(52), Span(4, 8)));
assert_eq!(btq.next().unwrap().0, N(12));
assert_eq!(btq.next().unwrap().0, N(32));
assert_eq!(btq.next().unwrap().0, N(52));
assert!(btq.next().is_none());

(多线程安全) 并行标记队列

use tokenizer_lib::{ParallelTokenQueue, Token, TokenReader, TokenSender, TokenTrait};

#[derive(PartialEq, Debug)]
struct Span(pub u32, pub u32);

#[derive(PartialEq, Debug)]
struct N(pub u32);

impl TokenTrait for N {}

let (mut sender, mut reader) = ParallelTokenQueue::new();
std::thread::spawn(move || {
    sender.push(Token(N(12), Span(0, 2)));
    sender.push(Token(N(32), Span(2, 4)));
    sender.push(Token(N(52), Span(4, 8)));
});

assert_eq!(reader.next().unwrap().0, N(12));
assert_eq!(reader.next().unwrap().0, N(32));
assert_eq!(reader.next().unwrap().0, N(52));
assert!(reader.next().is_none());

生成器标记队列

use tokenizer_lib::{GeneratorTokenQueue, GeneratorTokenQueueBuffer, Token, TokenReader, TokenSender, TokenTrait};

#[derive(PartialEq, Debug)]
struct N(pub u32);

impl TokenTrait for N {}

fn lexer(state: &mut u32, sender: &mut GeneratorTokenQueueBuffer<N, ()>) {
    *state += 1;
    match state {
        1..=3 => {
            sender.push(Token(N(*state * 2), ()));
        }
        _ => {}
    }
}

let mut reader = GeneratorTokenQueue::new(lexer, 0);

assert_eq!(reader.next().unwrap().0, N(2));
assert_eq!(reader.next().unwrap().0, N(4));
assert_eq!(reader.next().unwrap().0, N(6));
assert!(reader.next().is_none());

提供如 peek、peek_n 和 scan 等前瞻工具。还包括 expect_next 用于期望标记值和 conditional_next 用于根据谓词前进。