uscan

通用源代码扫描器

/!\ 进度中

功能

• 可配置关键词、符号和注释
• 处理嵌套多行注释
• 处理十进制（15）、十六进制（0xf 或 0xF）和二进制（0b1111）字面数字

用法

const LUA_CONFIG: ScannerConfig = ScannerConfig {
    keywords: &[
        "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in",
        "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while",
    ],
    symbols: &[
        "...", "..", "==", "~=", "<=", ">=", "+", "-", "*", "/", "%", "^", "#", "<", ">", "=", "(",
        ")", "{", "}", "[", "]", ";", ":", ",", ".",
    ],
    single_line_cmt: Some("--"),
    multi_line_cmt_start: Some("--[["),
    multi_line_cmt_end: Some("]]"),
};

let mut scanner_data = ScannerData::default();
let mut scanner = Scanner::default();
scanner.run(source_code, &LUA_CONFIG, &mut scanner_data)?;

=> 现在您可以在解析器中使用 ScannerData 结构来构建您的抽象语法树 (AST)

pub enum TokenType {
    Symbol(String),
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(String, Number),
    Keyword(String),
    Comment(String),
    // space
    Ignore,
    NewLine,
    Eof,
    Unknown,
}

pub struct ScannerData {
    /// complete source code
    pub source: Vec<char>,
    /// resulting list of tokens
    pub token_types: Vec<TokenType>,
    /// token start line in the source code
    pub token_lines: Vec<usize>,
    /// token start offset from its line beginning
    pub token_start: Vec<usize>,
    /// token length in characters
    /// not always = token value's length.
    /// for example for TokenType::StringLiteral("aa") the value length is 2 but the token length including the quotes is 4
    pub token_len: Vec<usize>,
}