WikiText Table Parser

用Rust编写的WikiText表格解析器。

我们还有一个Python绑定。

这个项目是做什么的？

WikiText是维基百科使用的特殊格式，大多数可用的wiki数据或处理工具忽略了表格数据。本项目实现了一个表格解析器，帮助用户在维基文本或wiki-dump中处理表格数据。

维基文本中的表格应该看起来像

{| class="wikitable"
|+ Caption text
|-
! Header text !! Header text !! Header text
|-
| Example || Example || Example
|-
| Example || Example || Example
|-
| Example || Example || Example
|}

还可以参考wikitext表格获取更多详细信息。

文档

Rust

安装

[dependencies]
wikitext_table_parser = "0.3.1"

使用示例

use std::env;
use std::fs::File;
use std::io::Read;
use wikitext_table_parser::parser::{Event, WikitextTableParser};
use wikitext_table_parser::tokenizer::{
    get_all_cell_text_special_tokens, get_all_table_special_tokens, Tokenizer,
};

fn main() {
    let args: Vec<String> = env::args().collect();
    let file_path = args[1].clone();

    // Attempt to open the file
    let mut file = match File::open(file_path) {
        Ok(file) => file,
        Err(_) => {
            eprintln!("Error opening the file.");
            return;
        }
    };

    // Read the contents of the file into a String
    let mut content = String::new();
    if let Err(_) = file.read_to_string(&mut content) {
        eprintln!("Error reading the file into a string.");
        return;
    }
    let table_tokenizer = Tokenizer::build(get_all_table_special_tokens());
    let cell_tokenizer = Tokenizer::build(get_all_cell_text_special_tokens());
    let wikitext_table_parser =
        WikitextTableParser::new(table_tokenizer, cell_tokenizer, &content, true);
    for event in wikitext_table_parser {
        match event {
            Event::TableStart {} => {
                println!("Table START!");
            }
            Event::TableStyle { text: table_style } => {
                println!("table style{:?}#", table_style);
            }
            Event::TableCaption { text } => {
                println!("table name{:?}#", text);
            }
            Event::RowStyle { text: row_style } => {
                println!("----- {:?} -----", row_style);
            }
            Event::ColStart { cell_type } =>{
                print!("{:?} ",cell_type);
            }
            Event::ColStyle { text: col_style } => {
                print!("style: {:?} -> ", col_style);
            }
            Event::ColEnd { text } => {
                println!("data: {:?}", text);
            }
            Event::TableEnd {} => {
                println!("Table END!");
            }
            _ => {}
        }
    }
}

Python

安装

1. 从发布下载wheel文件

2. 安装wheel

pip install wikitext_table_parser-xxx.whl

使用示例

import sys
from wikitext_table_parser import (
    WikitextTableParser,
    Tokenizer,
    Event,
    get_all_table_special_tokens,
    get_all_cell_text_special_tokens
)

table_tokens = get_all_table_special_tokens()
cell_tokens = get_all_cell_text_special_tokens()

table_tokenizer = Tokenizer(table_tokens)
cell_tokenizer = Tokenizer(cell_tokens)

test_case = open(sys.argv[-1]).read()

parser = WikitextTableParser(table_tokenizer, cell_tokenizer, test_case, True)
print(parser.tokens)

while (len(parser.tokens) > 0):
    parser.step()

for event in parser.event_log_queue:
    if isinstance(event, Event.TableStart):
        pass
    elif isinstance(event, Event.TableStyle):
        print("table style:", event.text)
    elif isinstance(event, Event.TableEnd):
        pass
    elif isinstance(event, Event.ColStart):
        print("col type:", event.cell_type)
    elif isinstance(event, Event.ColStyle):
        print("col style:", event.text)
    elif isinstance(event, Event.ColEnd):
        print("col data:", event.text)
        print("-"*20)
    elif isinstance(event, Event.TableCaptionStart):
        pass
    elif isinstance(event, Event.TableCaption):
        print("table caption:", event.text)
    elif isinstance(event, Event.RowStart):
        pass
    elif isinstance(event, Event.RowStyle):
        print("row style:", event.text)
    elif isinstance(event, Event.RowEnd):
        print("="*30)
    else:
        raise NotImplementedError(event)