gzb_binary_60

这是一个在磁盘上以二进制格式存储键值对的二进制格式

结构

我们使用标志来分隔数据部分，每个部分包含关于行开始和结束的元信息，以及关于键和值数据的元信息。

let key = vec![0,0,0,1];
let key_len = 4;
let key_len_num_as_bytes = vec![0,0,0,0,0,0,0,4]//bigINT u64 as bytes
let len_of_key_len_num_as_bytes = key_len_num_as_bytes.len();

let value = vec![0,0,0,2];
let value_len = 4;
let value_len_num_as_bytes = vec![0,0,0,0,0,0,0,4]//bigINT u64 as bytes
let len_of_value_len_num_as_bytes = value_len_num_as_bytes.len();


//this is vector of u8 values
0,1,0                           +       //start flag
len_of_key_len_num_as_bytes     +       
0,2,0                           +       //seprator flag
key_len_num_as_bytes            +
0,3,0                           +       //key start flag
key                             +       
0,4,0                           +       //seprator flag
len_of_value_len_num_as_bytes   +
0,5,0                           +       //seprator flag
value_len_num_as_bytes          +
0,6,0                           +       //value start flag
value                           +
0,7,0                                   //end flag

示例代码

use gzb_binary_69::{worker,Reader,PointerType,parser};
use std::time::Instant;

fn main() {

    let time_start = Instant::now();

    //-------------------------
    //parse test
    //-------------------------

    let mut parsed_1 = parser::writer::init(
        (1 as u64).to_be_bytes().to_vec(),
        String::from("value 12").as_bytes().to_vec(),
    ).unwrap();

    let mut parsed_2 = parser::writer::init(
        (2 as u64).to_be_bytes().to_vec(),
        String::from("value 23334").as_bytes().to_vec(),
    ).unwrap();

    let mut parsed_3 = parser::writer::init(
        (3 as u64).to_be_bytes().to_vec(),
        String::from("value 999999991").as_bytes().to_vec(),
    ).unwrap();

    //-------------------------
    // build primary buffer
    //-------------------------

    let mut collect = vec![];

    // add curropt data test
    if false{
        collect.append(&mut vec![0,0,0,0/*3*/,1,2/*5*/,0/*6*/,3/*7*/,0,0,0/*10*/]);
    }

    //add for empty space test
    if false {
        for _ in 0..99_999{
            collect.push(0);
        }
        println!("bytes alloted : {:?}",time_start.elapsed().as_millis());
    }

    //add for fill space test
    if false {
        collect.append(&mut parser::writer::init(
            (1 as u64).to_be_bytes().to_vec(),
            String::from("value").as_bytes().to_vec(),
        ).unwrap());
        for _ in 0..3{collect.push(0);}
        collect.append(&mut parser::writer::init(
            (2 as u64).to_be_bytes().to_vec(),
            String::from("value").as_bytes().to_vec(),
        ).unwrap());
        for _ in 0..100{collect.push(0);}
        println!("bytes alloted : {:?}",time_start.elapsed().as_millis());
    }

    //add for line quantity test
    if false {
        let mut index:u64 = 1;
        for _ in 0..1000{
            let mut build = parser::writer::init(
                index.to_be_bytes().to_vec(),
                String::from("value").as_bytes().to_vec(),
            ).unwrap();
            collect.append(&mut vec![0,0,0,0]);
            collect.append(&mut build);
            index += 1;
        }
        collect.append(&mut vec![0,0,0,0]);
        println!("lines alloted : {:?} {:?}",index,time_start.elapsed().as_millis());
    }

    //add for remove test
    if true{
        collect.append(&mut vec![0,0,0,0]);
        collect.append(&mut parsed_1);
        collect.append(&mut vec![0,0,0,0]);
        collect.append(&mut parsed_2);
        collect.append(&mut vec![0,0,0,0]);
        collect.append(&mut parsed_3);
        // collect.append(&mut vec![0,0,0,0]);
    }

    //-------------------------
    //build reader commands
    //-------------------------

    let mut r = Reader::with_capacity(1000000, 1000000);
    // r.get_values(true);

    //-------------------------
    //test commands
    //-------------------------

    //test find keys
    if true{
        r.find_key((1 as u64).to_be_bytes().to_vec());
        r.find_key((2 as u64).to_be_bytes().to_vec());
        r.find_key((3 as u64).to_be_bytes().to_vec());
    }

    //-------------------------
    //print raw buffer
    //-------------------------

    if false{workers::buff_print(&collect);}
    if false{
        println!("input bytes size : {:?} {:?}",collect.len(),time_start.elapsed().as_millis());
    }

    //-------------------------
    //make and push blocks from primary buffer
    //-------------------------
    let mut pool = vec![];
    let mut buffer = vec![];
    for i in collect{
        if &buffer.len() == &10000{
            &pool.push(buffer.clone());
            &buffer.clear();
            &buffer.push(i);
        } else {
            &buffer.push(i);
        }
    }
    if buffer.len() > 0{
        pool.push(buffer);
    }

    loop{
        if pool.len()==0{
            // println!("bytes pushed : {:?}",time_start.elapsed().as_millis());
            break;
        }
        let mut part = pool.remove(0);
        match &r.map(&mut part){
            Ok(_)=>{},
            Err(_)=>{}
        }
    }
    match &r.end(){
        Ok(_)=>{
            println!("map ended : {:?}",time_start.elapsed().as_millis());
        },
        Err(_)=>{}
    }

    //-------------------------
    //test edits 
    //-------------------------

    //test fill
    if false{
        println!("\nfill test==");
        let key = (6 as u64).to_be_bytes().to_vec();
        let value = String::from("value").as_bytes().to_vec();
        match &r.fill(key,value.len()){
            Ok(_)=>{
                println!("==fill successfull");
            },
            Err(_)=>{
                println!("\n==fill failed");
            }
        }
    }

    //test get
    if false{
        match &r.find(&(1 as u64).to_be_bytes().to_vec()){
            Ok(v)=>{
                match &r.clear(*v){
                    Ok(_)=>{
                        println!("\n>>> clear successfull");
                    },
                    Err(_)=>{
                        println!("!!! clear failed");
                    }
                }
            },
            Err(_)=>{
                println!("!!! find failed");
            }
        }
    }

    //test expand
    if true{
        if true{workers::print_pointers(&mut r);}
        match &r.expand(10){
            Ok(_)=>{
                println!("==expand successfull");
            },
            Err(_)=>{
                println!("==expand failed");
            }
        }
    }

    //-------------------------
    //debug reader map
    //-------------------------

    if true{workers::print_pointers(&mut r);}
    if false {workers::print_raw_pointers(&mut r);}
    if false{workers::print_last_100_pointers(&mut r);}

    if false{
        println!("final map len : {:?} {:?}",r.map.len(),time_start.elapsed().as_millis());
    }
    if false{
        println!("final end : {:?}",time_start.elapsed().as_millis());
    }
    if false{
        println!("{:?}",r.values);
    }
    if false{
        for i in r.values{
            println!("{:?}",String::from_utf8(i.1));
        }
    }

}