Rust-Warc

高性能且易于使用的Web存档(WARC)文件读取器

use rust_warc::WarcReader;

use std::io;

fn main() {
    // we're taking input from stdin here, but any BufRead will do
    let stdin = io::stdin();
    let handle = stdin.lock();

    let warc = WarcReader::new(handle);

    let mut response_counter = 0;
    let mut response_size = 0;

    for item in warc {
        let record = item.unwrap(); // could be IO/malformed error

        // header names are case insensitive
        if record.header.get(&"WARC-Type".into()) == Some(&"response".into()) {
            response_counter += 1;
            response_size += record.content.len();
        }
    }

    println!("response records: {}", response_counter);
    println!("response size: {} MiB", response_size >> 20);
}

lib.rs:

高性能Web存档(WARC)文件解析器

WarcReader遍历来自[BufRead]输入的WarcRecords。

性能应该相当不错，单核CPU上约为~500MiB/s。

使用方法

use rust_warc::WarcReader;

fn main() {
    // we're taking input from stdin here, but any BufRead will do
    let stdin = std::io::stdin();
    let handle = stdin.lock();

    let mut warc = WarcReader::new(handle);

    let mut response_counter = 0;
    for item in warc {
        let record = item.expect("IO/malformed error");

        // header names are case insensitive
        if record.header.get(&"WARC-Type".into()) == Some(&"response".into()) {
            response_counter += 1;
        }
    }

    println!("# response records: {}", response_counter);
}