Myval - a lightweight Apache Arrow data frame for Rust

What is Myval?

Why not a bear-name?

But there is Polars?

let polars_df = polars::frame::DataFrame::from(myval_df);let myval_df = myval::DataFrame::from(polars_df);

Some tricks

IPC

let df = myval::DataFrame::from_ipc_block(&buf).unwrap();

let buf = df.into_ipc_block().unwrap();

Overriding data types

use myval::{DataType,TimeUnit};df.set_data_type("time",DataType::Timestamp(TimeUnit::Nanosecond,None)).unwrap();

Parsing numbers from strings

df.parse::<f64>("value").unwrap();

Basic in-place math

df.add("col",1_000i64).unwrap();df.sub("col",1_000i64).unwrap();df.mul("col",1_000i64).unwrap();df.div("col",1_000i64).unwrap();

Custom in-place transformations

df.apply("time", |time| time.map(|t:i64| t /1_000)).unwrap();

Horizontal join

df.join(df2).unwrap();

Concatenation

let merged = myval::concat(&[&df1,&df2,&df3]).unwrap();

Set column ordering

df.set_ordering(&["voltage","temp1","temp2","temp3"]);

From/to JSON

// create Object value from a data frame, converted to serde_json::Maplet val = serde_json::Value::Object(df.to_json_map().unwrap());// define JSON parserletmut parser = myval::convert::json::Parser::new().with_type_mapping("name",DataType::LargeUtf8);// add more columns if requiredparser = parser.with_type_mapping("time",DataType::Int64);parser = parser.with_type_mapping("status",DataType::Int32);let parsed_df = parser.parse_value(val).unwrap();

Others

Working with databases

Fetching data from a database

use futures::stream::TryStreamExt;let pool =PgPoolOptions::new().connect("postgres://postgres:welcome@localhost/postgres").await.unwrap();let max_size =100_000;letmut stream = myval::db::postgres::fetch("select * from test".to_owned(),Some(max_size), pool.clone());// the stream returns data frames one by one with max data frame size (in// bytes) = max_sizewhileletSome(df) = stream.try_next().await.unwrap(){// do some stuff}

Pushing data into a database

Server

let df =DataFrame::from_ipc_block(payload).unwrap();// The first received data frame must have "database" field in its schema// metadata. Next data frames can go without it.ifletSome(dbparams) = df.metadata().get("database"){let params: myval::db::postgres::Params = serde_json::from_str(dbparams).unwrap();let processed_rows:usize = myval::db::postgres::push(&df,&params,&pool).await.unwrap();}

Client

use serde_json::json;letmut df = myval::DataFrame::from(polars_df);df.metadata_mut().insert(// set "database" metadata field"database".to_owned(),    serde_json::to_string(&json!({// table, required"table":"test",// PostgreSQL schema, optional"postgres":{"schema":"public"},// keys, required if the table has got keys/unique indexes"keys":["id"],// some field parameters"fields":{// another way to declare a key field//"id": { "key": true },// the following data frame columns contain strings which must be// sent to the database as JSON (for json/jsonb PostgreSQL types)"data1":{"json":true},"data2":{"json":true}}}))?,);// send the data frame to the server in a single or multiple chunks/blocks

PostgreSQL types supported

General limitations

About