Jun 5, 2023 · May 13, 2023 · May 15, 2023 · May 15, 2023 · May 15, 2023 · May 15, 2023
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
  workflow_dispatch:
    inputs:
      packageVersion:
        default: "2.4.4"
        default: "2.4.8"

 jobs:
  build:
    strategy:
      matrix:
        os: ["ubuntu-22.04"]
        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        for pg in {11..15}; do
          export PACKAGE_VERSION=${{ inputs.packageVersion }}
          export PGVERSION=${pg}
          export ARCH=amd64

          if [[ $(arch) == "x86_64" ]]; then
            export ARCH=amd64
          else
            export ARCH=arm64
          fi

          mkdir -p target/release/pgml-pg${pg}/DEBIAN
          (cat control | envsubst) > target/release/pgml-pg${pg}/DEBIAN/control
          dpkg-deb --root-owner-group --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-amd64.deb
          dpkg-deb --root-owner-group --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb

          deb-s3 upload \
            --bucket apt.postgresml.org \
            postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-amd64.deb \
            postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb \
            --codename $(lsb_release -cs)
        done
diff --git a/pgml-dashboard/Cargo.toml b/pgml-dashboard/Cargo.toml
 [package]
 name = "pgml-dashboard"
 version = "2.4.4"
 version = "2.4.8"
 edition = "2021"
 authors = ["PostgresML <team@postgresml.org>"]
 license = "MIT"
 tokio = { version = "1", features = ["full"] }
 yaml-rust = "0.4"
 zoomies = { git="https://github.com/HyperparamAI/zoomies.git", branch="master" }
 pgvector = { version = "0.2.0", features = [ "sqlx", "postgres" ] }

 [build-dependencies]
 md5 = "0.7"
diff --git a/pgml-dashboard/content/docs/guides/setup/developers.md b/pgml-dashboard/content/docs/guides/setup/developers.md

 === "Output"

 ```
 postgres=# select pgml.version();
      version
 -------------------
 2.4.4
 (1 row)
 ```

 ===
 ```
 postgres=# select pgml.version();
      version
 -------------------
 2.4.8
 (1 row)
 ```

 Basic extension usage:

diff --git a/pgml-dashboard/src/lib.rs b/pgml-dashboard/src/lib.rs
 pub mod forms;
 pub mod guards;
 pub mod models;
 pubmod responses;
 pubmod templates;
 pubmod utils;
 mod responses;
 mod templates;
 mod utils;

 use crate::templates::{
    DeploymentsTab, Layout, ModelsTab, NotebooksTab, ProjectsTab, SnapshotsTab, UploaderTab,
 use responses::{BadRequest, Error, ResponseOk};
 use sqlx::Executor;

 #[derive(Debug, Default, Clone)]
 pub struct ClustersSettings {
    pub max_connections: u32,
    pub idle_timeout: u64,
    pub min_connections: u32,
 }

 /// This struct contains information specific to the cluster being displayed in the dashboard.
 ///
 /// The dashboard is built to manage multiple clusters, but the server itself by design is stateless.
 }

 impl Clusters {
    pub fn add(&self, cluster_id: i64, database_url: &str) -> anyhow::Result<PgPool> {
    pub fn add(
        &self,
        cluster_id: i64,
        database_url: &str,
        settings: ClustersSettings,
    ) -> anyhow::Result<PgPool> {
        let mut pools = self.pools.lock();

        let pool = PgPoolOptions::new()
            .max_connections(5)
            .idle_timeout(std::time::Duration::from_millis(15_000))
            .min_connections(0)
            .max_connections(settings.max_connections)
            .idle_timeout(std::time::Duration::from_millis(settings.idle_timeout))
            .min_connections(settings.min_connections)
            .after_connect(|conn, _meta| {
                Box::pin(async move {
                    conn.execute("SET application_name = 'pgml_dashboard';")
    let sql = templates::Sql::new(
        cluster.pool(),
        &format!("SELECT * FROM {} LIMIT 10", table_name),
        true,
    )
    .await
    .unwrap();
diff --git a/pgml-dashboard/src/main.rs b/pgml-dashboard/src/main.rs
    markdown::SearchIndex::build().await.unwrap();

    let clusters = pgml_dashboard::Clusters::new();
    let settings = pgml_dashboard::ClustersSettings {
        min_connections: 0,
        max_connections: 5,
        idle_timeout: 15_000,
    };

    clusters
        .add(-1, &pgml_dashboard::guards::default_database_url())
        .add(
            -1,
            &pgml_dashboard::guards::default_database_url(),
            settings,
        )
        .unwrap();

    pgml_dashboard::migrate(&clusters.get(-1).unwrap())

    async fn rocket() -> Rocket<Build> {
        dotenv::dotenv().ok();
        let max_connections = 5;
        let min_connections = 1;
        let idle_timeout = 15_000;

        let clusters = Clusters::new();
        clusters
diff --git a/pgml-dashboard/src/models.rs b/pgml-dashboard/src/models.rs
    pub async fn render(&mut self, pool: &PgPool) -> anyhow::Result<()> {
        let cell_type: CellType = self.cell_type.into();

        let rendering = match cell_type {
        let(rendering, execution_time) = match cell_type {
            CellType::Sql => {
                let queries= self.contents.split(";");
                let queries: Vec<&str>= self.contents.split(';').filter(|q| !q.trim().is_empty()).collect();
                let mut rendering = String::new();
                let mut total_execution_duration = std::time::Duration::default();
                let render_individual_execution_duration = queries.len() > 1;

                for query in queries {
                    if query.trim().is_empty() {
                        continue;
                    }

                    let result = match templates::Sql::new(pool, query).await {
                        Ok(sql) => sql.render_once()?,
                    let result = match templates::Sql::new(pool, query, render_individual_execution_duration).await {
                        Ok(sql) => {
                            total_execution_duration += sql.execution_duration;
                            sql.render_once()?
                        },
                        Err(err) => templates::SqlError {
                            error: format!("{:?}", err),
                        }
                    rendering.push_str(&result);
                }

                rendering
                let execution_time = PgInterval{
                    months: 0,
                    days: 0,
                    microseconds: total_execution_duration.as_micros().try_into().unwrap_or(0)
                };
                (rendering, Some(execution_time))
            }

            CellType::Markdown => {
                    front_matter_delimiter: None,
                };

                format!(
 (format!(
                    "<div class=\"markdown-body\">{}</div>",
                    markdown_to_html(&self.contents, &options)
                )
                ), None)
            }
        };

        sqlx::query!(
            "UPDATE pgml.notebook_cells SET rendering = $1WHERE id = $2",
            "UPDATE pgml.notebook_cells SET rendering = $1, execution_time = $2WHERE id = $3",
            rendering,
            execution_time,
            self.id
        )
        .execute(pool)
        .await?;

        self.execution_time = execution_time;
        self.rendering = Some(rendering);

        Ok(())
diff --git a/pgml-dashboard/src/templates/mod.rs b/pgml-dashboard/src/templates/mod.rs
 pub struct Sql {
    pub columns: Vec<String>,
    pub rows: Vec<Vec<String>>,
    pub execution_duration: std::time::Duration,
    pub render_execution_duration: bool,
 }

 impl Sql {
    pub async fn new(pool: &PgPool, query: &str) -> anyhow::Result<Sql> {
    pub async fn new(pool: &PgPool, query: &str, render_execution_duration: bool) -> anyhow::Result<Sql> {
        let prepared_stmt = pool.prepare(query).await?;
        let cols = prepared_stmt.columns();


        cols.iter().for_each(|c| columns.push(c.name().to_string()));

        let now = std::time::Instant::now();
        let result = prepared_stmt.query().fetch_all(pool).await?;
        let execution_duration = now.elapsed();

        for row in result.iter() {
            let mut values = Vec::new();
                        serde_json::to_string(&value)?
                    }

                    "vector" => {
                        let value: pgvector::Vector = row.try_get(i)?;
                        format!("{:?}", value.to_vec())
                    }

                    unknown => {
                        // TODO
                        // Implement everything here: https://docs.rs/sqlx/latest/sqlx/postgres/types/index.html
            rows.push(values);
        }

        Ok(Sql { columns, rows })
        Ok(Sql { columns, rows, execution_duration, render_execution_duration })
    }
 }

diff --git a/pgml-dashboard/src/utils.rs b/pgml-dashboard/src/utils.rs
 pub fn format_microseconds(microseconds: f64) -> String {
    if microseconds >= 1000000. {
        format!("{}s", microseconds / 1000000.)
    } else if microseconds >= 1000. {
        format!("{}ms", microseconds / 1000.)
    } else {
        format!("{}μs", microseconds)
    }
 }
diff --git a/pgml-dashboard/templates/content/dashboard/panels/cell.html b/pgml-dashboard/templates/content/dashboard/panels/cell.html
      <div class="notebook-duration">
        <div class="markdown-body">
          <div class="flex flex-row-reverse">
            <code>TODO</code>
            <code>Time: <%= crate::utils::format_microseconds(cell.execution_time.unwrap().microseconds as f64) %></code>
          </div>
        </div>
      </div>
diff --git a/pgml-dashboard/templates/content/sql.html b/pgml-dashboard/templates/content/sql.html
      <% } %>
    </tbody>
  </table>
  <% if render_execution_duration { %>
    <code>Time: <%= crate::utils::format_microseconds(execution_duration.as_micros() as f64) %></code>
  <% } %>
 </div>

 <% } %>
diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock
diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml
 [package]
 name = "pgml"
 version = "2.4.4"
 version = "2.4.8"
 edition = "2021"

 [lib]
diff --git a/pgml-extension/docker/Cargo.toml.no-python b/pgml-extension/docker/Cargo.toml.no-python

 [features]
 default = ["pg15"]
 pg10 = ["pgrx/pg10", "pgrx-tests/pg10" ]
 pg11 = ["pgrx/pg11", "pgrx-tests/pg11" ]
 pg12 = ["pgrx/pg12", "pgrx-tests/pg12" ]
 pg13 = ["pgrx/pg13", "pgrx-tests/pg13" ]
 pg14 = ["pgrx/pg14", "pgrx-tests/pg14" ]
 pg14 = ["pgrx/pg15", "pgrx-tests/pg15" ]
 pg15 = ["pgrx/pg15", "pgrx-tests/pg15" ]
 pg_test = []
 python = ["pyo3"]
 cuda = ["xgboost/cuda", "lightgbm/cuda"]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,13 +4,13 @@ on:
		workflow_dispatch:
		inputs:
		packageVersion:
		default: "2.4.4"
		default: "2.4.8"

		jobs:
		build:
		strategy:
		matrix:
		os: ["ubuntu-22.04"]
		os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
		runs-on: ${{ matrix.os }}
		defaults:
		run:
Expand DownExpand Up		@@ -130,14 +130,19 @@ jobs:
		for pg in {11..15}; do
		export PACKAGE_VERSION=${{ inputs.packageVersion }}
		export PGVERSION=${pg}
		export ARCH=amd64

		if [[ $(arch) == "x86_64" ]]; then
		export ARCH=amd64
		else
		export ARCH=arm64
		fi

		mkdir -p target/release/pgml-pg${pg}/DEBIAN
		(cat control \| envsubst) > target/release/pgml-pg${pg}/DEBIAN/control
		dpkg-deb --root-owner-group --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-amd64.deb
		dpkg-deb --root-owner-group --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb

		deb-s3 upload \
		--bucket apt.postgresml.org \
		postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-amd64.deb \
		postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb \
		--codename $(lsb_release -cs)
		done
Original file line number	Diff line number	Diff line change
		@@ -1,6 +1,6 @@
		[package]
		name = "pgml-dashboard"
		version = "2.4.4"
		version = "2.4.8"
		edition = "2021"
		authors = ["PostgresML <team@postgresml.org>"]
		license = "MIT"
Expand DownExpand Up		@@ -43,6 +43,7 @@ time = "0.3"
		tokio = { version = "1", features = ["full"] }
		yaml-rust = "0.4"
		zoomies = { git="https://github.com/HyperparamAI/zoomies.git", branch="master" }
		pgvector = { version = "0.2.0", features = [ "sqlx", "postgres" ] }

		[build-dependencies]
		md5 = "0.7"
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -123,15 +123,13 @@ SELECT pgml.version();

		=== "Output"

		```
		postgres=# select pgml.version();
		version
		-------------------
		2.4.4
		(1 row)
		```

		===
		```
		postgres=# select pgml.version();
		version
		-------------------
		2.4.8
		(1 row)
		```

		Basic extension usage:

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,9 +16,9 @@ pub mod fairings;
		pub mod forms;
		pub mod guards;
		pub mod models;
		pubmod responses;
		pubmod templates;
		pubmod utils;
		mod responses;
		mod templates;
		mod utils;

		use crate::templates::{
		DeploymentsTab, Layout, ModelsTab, NotebooksTab, ProjectsTab, SnapshotsTab, UploaderTab,
Expand All		@@ -28,6 +28,13 @@ use guards::Cluster;
		use responses::{BadRequest, Error, ResponseOk};
		use sqlx::Executor;

		#[derive(Debug, Default, Clone)]
		pub struct ClustersSettings {
		pub max_connections: u32,
		pub idle_timeout: u64,
		pub min_connections: u32,
		}

		/// This struct contains information specific to the cluster being displayed in the dashboard.
		///
		/// The dashboard is built to manage multiple clusters, but the server itself by design is stateless.
Expand All		@@ -49,13 +56,18 @@ pub struct Clusters {
		}

		impl Clusters {
		pub fn add(&self, cluster_id: i64, database_url: &str) -> anyhow::Result<PgPool> {
		pub fn add(
		&self,
		cluster_id: i64,
		database_url: &str,
		settings: ClustersSettings,
		) -> anyhow::Result<PgPool> {
		let mut pools = self.pools.lock();

		let pool = PgPoolOptions::new()
		.max_connections(5)
		.idle_timeout(std::time::Duration::from_millis(15_000))
		.min_connections(0)
		.max_connections(settings.max_connections)
		.idle_timeout(std::time::Duration::from_millis(settings.idle_timeout))
		.min_connections(settings.min_connections)
		.after_connect(\|conn, _meta\| {
		Box::pin(async move {
		conn.execute("SET application_name = 'pgml_dashboard';")
Expand DownExpand Up		@@ -511,6 +523,7 @@ pub async fn uploaded_index(cluster: Cluster, table_name: &str) -> ResponseOk {
		let sql = templates::Sql::new(
		cluster.pool(),
		&format!("SELECT * FROM {} LIMIT 10", table_name),
		true,
		)
		.await
		.unwrap();
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -118,8 +118,18 @@ async fn main() {
		markdown::SearchIndex::build().await.unwrap();

		let clusters = pgml_dashboard::Clusters::new();
		let settings = pgml_dashboard::ClustersSettings {
		min_connections: 0,
		max_connections: 5,
		idle_timeout: 15_000,
		};

		clusters
		.add(-1, &pgml_dashboard::guards::default_database_url())
		.add(
		-1,
		&pgml_dashboard::guards::default_database_url(),
		settings,
		)
		.unwrap();

		pgml_dashboard::migrate(&clusters.get(-1).unwrap())
Expand DownExpand Up		@@ -159,6 +169,9 @@ mod test {

		async fn rocket() -> Rocket<Build> {
		dotenv::dotenv().ok();
		let max_connections = 5;
		let min_connections = 1;
		let idle_timeout = 15_000;

		let clusters = Clusters::new();
		clusters
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -288,18 +288,19 @@ impl Cell {
		pub async fn render(&mut self, pool: &PgPool) -> anyhow::Result<()> {
		let cell_type: CellType = self.cell_type.into();

		let rendering = match cell_type {
		let(rendering, execution_time) = match cell_type {
		CellType::Sql => {
		let queries= self.contents.split(";");
		let queries: Vec<&str>= self.contents.split(';').filter(\|q\| !q.trim().is_empty()).collect();
		let mut rendering = String::new();
		let mut total_execution_duration = std::time::Duration::default();
		let render_individual_execution_duration = queries.len() > 1;

		for query in queries {
		if query.trim().is_empty() {
		continue;
		}

		let result = match templates::Sql::new(pool, query).await {
		Ok(sql) => sql.render_once()?,
		let result = match templates::Sql::new(pool, query, render_individual_execution_duration).await {
		Ok(sql) => {
		total_execution_duration += sql.execution_duration;
		sql.render_once()?
		},
		Err(err) => templates::SqlError {
		error: format!("{:?}", err),
		}
Expand All		@@ -309,7 +310,12 @@ impl Cell {
		rendering.push_str(&result);
		}

		rendering
		let execution_time = PgInterval{
		months: 0,
		days: 0,
		microseconds: total_execution_duration.as_micros().try_into().unwrap_or(0)
		};
		(rendering, Some(execution_time))
		}

		CellType::Markdown => {
Expand All		@@ -327,21 +333,23 @@ impl Cell {
		front_matter_delimiter: None,
		};

		format!(
		(format!(
		"<div class=\"markdown-body\">{}</div>",
		markdown_to_html(&self.contents, &options)
		)
		), None)
		}
		};

		sqlx::query!(
		"UPDATE pgml.notebook_cells SET rendering = $1WHERE id = $2",
		"UPDATE pgml.notebook_cells SET rendering = $1, execution_time = $2WHERE id = $3",
		rendering,
		execution_time,
		self.id
		)
		.execute(pool)
		.await?;

		self.execution_time = execution_time;
		self.rendering = Some(rendering);

		Ok(())
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -141,10 +141,12 @@ pub struct Undo {
		pub struct Sql {
		pub columns: Vec<String>,
		pub rows: Vec<Vec<String>>,
		pub execution_duration: std::time::Duration,
		pub render_execution_duration: bool,
		}

		impl Sql {
		pub async fn new(pool: &PgPool, query: &str) -> anyhow::Result<Sql> {
		pub async fn new(pool: &PgPool, query: &str, render_execution_duration: bool) -> anyhow::Result<Sql> {
		let prepared_stmt = pool.prepare(query).await?;
		let cols = prepared_stmt.columns();

Expand All		@@ -153,7 +155,9 @@ impl Sql {

		cols.iter().for_each(\|c\| columns.push(c.name().to_string()));

		let now = std::time::Instant::now();
		let result = prepared_stmt.query().fetch_all(pool).await?;
		let execution_duration = now.elapsed();

		for row in result.iter() {
		let mut values = Vec::new();
Expand DownExpand Up		@@ -267,6 +271,11 @@ impl Sql {
		serde_json::to_string(&value)?
		}

		"vector" => {
		let value: pgvector::Vector = row.try_get(i)?;
		format!("{:?}", value.to_vec())
		}

		unknown => {
		// TODO
		// Implement everything here: https://docs.rs/sqlx/latest/sqlx/postgres/types/index.html
Expand All		@@ -280,7 +289,7 @@ impl Sql {
		rows.push(values);
		}

		Ok(Sql { columns, rows })
		Ok(Sql { columns, rows, execution_duration, render_execution_duration })
		}
		}

Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,9 @@
		pub fn format_microseconds(microseconds: f64) -> String {
		if microseconds >= 1000000. {
		format!("{}s", microseconds / 1000000.)
		} else if microseconds >= 1000. {
		format!("{}ms", microseconds / 1000.)
		} else {
		format!("{}μs", microseconds)
		}
		}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -75,7 +75,7 @@
		<div class="notebook-duration">
		<div class="markdown-body">
		<div class="flex flex-row-reverse">
		<code>TODO</code>
		<code>Time: <%= crate::utils::format_microseconds(cell.execution_time.unwrap().microseconds as f64) %></code>
		</div>
		</div>
		</div>
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -24,6 +24,9 @@
		<% } %>
		</tbody>
		</table>
		<% if render_execution_duration { %>
		<code>Time: <%= crate::utils::format_microseconds(execution_duration.as_micros() as f64) %></code>
		<% } %>
		</div>

		<% } %>