#lexer #tokenizer #parallel #splitter #sentence

condex

通过简单的条件表达式提取标记

1个稳定版本

1.0.0 2021年11月1日

#284 in 解析工具

MIT/Apache

12KB
244

Condex


通过简单的条件表达式提取标记。

CI Crates.io Licensed Twitter

| 文档 | 最新日志 |

[dependencies]
condex = "1.0.0"

示例


    #[derive(Debug, Clone, Copy, PartialEq)]
    enum Token {
        TagName,
        NameType,
        Value,
        AllInOne,
    }
    impl TokenKind for Token {}

    let mut builder = CondexBuilder::new(&[
        (Token::TagName, &["@-("]),
        (Token::NameType, &["[(,]  -  :  - [,=]"]),
        (Token::Value, &["=-[,)]"]),
        (Token::AllInOne, &["@-(", "[(,]  -  :  - [,=]", "=-[,)]"]),
    ]);

    let source = "@hello-man(name: type = value, name2: type2, name3: type3 = value3)";

    for (i, c) in source.char_indices() {
        builder.test(c, i);
    }
    let finals = builder.finalize_with_source(source);
    eprintln!("{:#?}", finals);
    

依赖项

~1.5MB
~25K SLoC