- Notifications
You must be signed in to change notification settings - Fork732
Open
Milestone
Description
The spec is:
GET /v1/list_datasets?function_name={function_name}&limit={limit}&offset={offset}to replicate this existing API request shape:
tensorzero/tensorzero-core/src/db/datasets.rs
Lines 198 to 209 inbb538de
| #[derive(Deserialize, ts_rs::TS)] | |
| #[cfg_attr(test, ts(export, optional_fields))] | |
| pubstructGetDatasetMetadataParams{ | |
| /// Only select datasets matching a specific function. | |
| pubfunction_name:Option<String>, | |
| /// The maximum number of datasets to return. | |
| publimit:Option<u32>, | |
| /// The number of datasets to skip before starting to return results. | |
| puboffset:Option<u32>, | |
| } |
returningListDatasetsResponse:
// this is newpubstructListDatasetsResponse{/// List of dataset metadata.pubdatasets:Vec<DatasetMetadata>,}// this is unchangedpubstructDatasetMetadata{/// The name of the dataset.pubdataset_name:String,/// The total number of datapoints in the dataset.pubcount:u32,/// The timestamp of the last update (ISO 8601 format).publast_updated:String,}
This will run the same query to list existing datasets ordering bylast_updated (all unchanged):
tensorzero/tensorzero-core/src/db/clickhouse/dataset_queries.rs
Lines 239 to 272 inbb538de
| let query =format!( | |
| r" | |
| SELECT | |
| dataset_name, | |
| toUInt32(sum(count)) AS count, | |
| formatDateTime(max(last_updated), '%Y-%m-%dT%H:%i:%SZ') AS last_updated | |
| FROM ( | |
| SELECT | |
| dataset_name, | |
| toUInt32(count()) AS count, | |
| max(updated_at) AS last_updated | |
| FROM ChatInferenceDatapoint | |
| FINAL | |
| WHERE staled_at IS NULL | |
| {function_where_clause} | |
| GROUP BY dataset_name | |
| UNION ALL | |
| SELECT | |
| dataset_name, | |
| toUInt32(count()) AS count, | |
| max(updated_at) AS last_updated | |
| FROM JsonInferenceDatapoint | |
| FINAL | |
| WHERE staled_at IS NULL | |
| {function_where_clause} | |
| GROUP BY dataset_name | |
| ) | |
| GROUP BY dataset_name | |
| ORDER BY last_updated DESC | |
| {limit_clause} | |
| {offset_clause} | |
| FORMAT JSONEachRow | |
| " | |
| ); |
We'll have rust and python implementations:
deflist_datasets(self,*,function_name:str|None=None,limit:int|None=None,offset:int|None=None)->ListDatasetsResponse: ...