Dec 20, 2023 · Dec 19, 2023 · Dec 19, 2023 · Dec 20, 2023 · Dec 20, 2023 · Dec 19, 2023
diff --git a/pgml-dashboard/.gitignore b/pgml-dashboard/.gitignore
 .DS_Store
 .DS_Store/
 node_modules
 cms/
diff --git a/pgml-dashboard/src/api/cms.rs b/pgml-dashboard/src/api/cms.rs
    utils::config,
 };

 use serde::{Deserialize, Serialize};

 lazy_static! {
    static ref BLOG: Collection = Collection::new("Blog", true);
    static ref CAREERS: Collection = Collection::new("Careers", true);
    static ref DOCS: Collection = Collection::new("Docs", false);
 }

 #[derive(Debug, Serialize, Deserialize)]
 pub struct Document {
    /// The absolute path on disk
    pub path: PathBuf,
    pub description: Option<String>,
    pub image: Option<String>,
    pub title: String,
    pub toc_links: Vec<TocLink>,
    pub html: String,
 }

 impl Document {
    pub async fn from_path(path: &PathBuf) -> anyhow::Result<Document> {
        let contents = tokio::fs::read_to_string(&path).await?;

        let parts = contents.split("---").collect::<Vec<&str>>();

        let (description, contents) = if parts.len() > 1 {
            match YamlLoader::load_from_str(parts[1]) {
                Ok(meta) => {
                    if meta.len() == 0 || meta[0].as_hash().is_none() {
                        (None, contents)
                    } else {
                        let description: Option<String> = match meta[0]["description"].is_badvalue()
                        {
                            true => None,
                            false => Some(meta[0]["description"].as_str().unwrap().to_string()),
                        };
                        (description, parts[2..].join("---").to_string())
                    }
                }
                Err(_) => (None, contents),
            }
        } else {
            (None, contents)
        };

        // Parse Markdown
        let arena = Arena::new();
        let spaced_contents = crate::utils::markdown::gitbook_preprocess(&contents);
        let root = parse_document(&arena, &spaced_contents, &crate::utils::markdown::options());

        // Title of the document is the first (and typically only) <h1>
        let title = crate::utils::markdown::get_title(root).unwrap();
        let toc_links = crate::utils::markdown::get_toc(root).unwrap();
        let image = crate::utils::markdown::get_image(root);
        crate::utils::markdown::wrap_tables(root, &arena).unwrap();

        // MkDocs, gitbook syntax support, e.g. tabs, notes, alerts, etc.
        crate::utils::markdown::mkdocs(root, &arena).unwrap();

        // Style headings like we like them
        let mut plugins = ComrakPlugins::default();
        let headings = crate::utils::markdown::MarkdownHeadings::new();
        plugins.render.heading_adapter = Some(&headings);
        plugins.render.codefence_syntax_highlighter =
            Some(&crate::utils::markdown::SyntaxHighlighter {});

        let mut html = vec![];
        format_html_with_plugins(
            root,
            &crate::utils::markdown::options(),
            &mut html,
            &plugins,
        )
        .unwrap();
        let html = String::from_utf8(html).unwrap();

        let document = Document {
            path: path.to_owned(),
            description,
            image,
            title,
            toc_links,
            html,
        };
        Ok(document)
    }
 }

 /// A Gitbook collection of documents
 #[derive(Default)]
 struct Collection {

    pub async fn get_asset(&self, path: &str) -> Option<NamedFile> {
        info!("get_asset: {} {path}", self.name);

        NamedFile::open(self.asset_dir.join(path)).await.ok()
    }

    pub async fn get_content(
        &self,
        mut path: PathBuf,
 cluster: &Cluster,
 doc_type: &str,
        origin: &Origin<'_>,
    ) -> Result<ResponseOk, Status> {
        info!("get_content: {} | {path:?}", self.name);

    ) -> Document {
        if origin.path().ends_with("/") {
            path = path.join("README");
        }

        let path =self.root_dir.join(format!("{}.md", path.to_string_lossy()));
        let path =PathBuf::from(format!("cms/{}/{}.json",doc_type,path.display()));

        self.render(&path, cluster, self).await
        let content = std::fs::read_to_string(path).unwrap();
        serde_json::from_str(&content).unwrap()
    }

    /// Create an index of the Collection based on the SUMMARY.md from Gitbook.
        Ok(links)
    }

    async fn render<'a>(
        &self,
        path: &'a PathBuf,
        cluster: &Cluster,
        collection: &Collection,
    ) -> Result<ResponseOk, Status> {
        // Read to string0
        let contents = match tokio::fs::read_to_string(&path).await {
            Ok(contents) => {
                info!("loading markdown file: '{:?}", path);
                contents
            }
            Err(err) => {
                warn!("Error parsing markdown file: '{:?}' {:?}", path, err);
                return Err(Status::NotFound);
            }
        };
        let parts = contents.split("---").collect::<Vec<&str>>();
        let (description, contents) = if parts.len() > 1 {
            match YamlLoader::load_from_str(parts[1]) {
                Ok(meta) => {
                    if !meta.is_empty() {
                        let meta = meta[0].clone();
                        if meta.as_hash().is_none() {
                            (None, contents.to_string())
                        } else {
                            let description: Option<String> = match meta["description"]
                                .is_badvalue()
                            {
                                true => None,
                                false => Some(meta["description"].as_str().unwrap().to_string()),
                            };

                            (description, parts[2..].join("---").to_string())
                        }
                    } else {
                        (None, contents.to_string())
                    }
                }
                Err(_) => (None, contents.to_string()),
            }
        } else {
            (None, contents.to_string())
        };

        // Parse Markdown
        let arena = Arena::new();
        let root = parse_document(&arena, &contents, &crate::utils::markdown::options());

        // Title of the document is the first (and typically only) <h1>
        let title = crate::utils::markdown::get_title(root).unwrap();
        let toc_links = crate::utils::markdown::get_toc(root).unwrap();
        let image = crate::utils::markdown::get_image(root);
        crate::utils::markdown::wrap_tables(root, &arena).unwrap();

        // MkDocs syntax support, e.g. tabs, notes, alerts, etc.
        crate::utils::markdown::mkdocs(root, &arena).unwrap();

        // Style headings like we like them
        let mut plugins = ComrakPlugins::default();
        let headings = crate::utils::markdown::MarkdownHeadings::new();
        plugins.render.heading_adapter = Some(&headings);
        plugins.render.codefence_syntax_highlighter =
            Some(&crate::utils::markdown::SyntaxHighlighter {});

        // Render
        let mut html = vec![];
        format_html_with_plugins(
            root,
            &crate::utils::markdown::options(),
            &mut html,
            &plugins,
        )
        .unwrap();
        let html = String::from_utf8(html).unwrap();

        // Handle navigation
        // TODO organize this functionality in the collection to cleanup
        let index: Vec<IndexLink> = self
            .index
    // Sets specified index as currently viewed.
    pub fn open_index(&self, path: PathBuf) -> Vec<IndexLink> {
        self.index
            .clone()
            .iter_mut()
            .map(|nav_link| {
                let mut nav_link = nav_link.clone();
                nav_link.should_open(path);
                nav_link.should_open(&path);
                nav_link
            })
            .collect();
            .collect()
    }

    async fn render<'a>(&self, doc: Document, cluster: &Cluster) -> Result<ResponseOk, Status> {
        let index = self.open_index(doc.path);

        let user = if cluster.context.user.is_anonymous() {
            None
        } else {
            Some(cluster.context.user.clone())
        };

        let mut layout = crate::templates::Layout::new(&title, Some(cluster));
        if let Some(image) = image {
            // translate relative url into absolute for head social sharing
            let parts = image.split(".gitbook/assets/").collect::<Vec<&str>>();
            let image_path = collection.url_root.join(".gitbook/assets").join(parts[1]);
            layout.image(config::asset_url(image_path.to_string_lossy()).as_ref());
        let mut layout = crate::templates::Layout::new(&doc.title, Some(cluster));
        if let Some(image) = doc.image {
            layout.image(&config::asset_url(image.into()));
        }
        if let Some(description) = &description {
        if let Some(description) = &doc.description {
            layout.description(description);
        }
        if let Some(user) = &user {
            layout.user(user);
        }



        let layout = layout
            .nav_title(&self.name)
            .nav_links(&index)
            .toc_links(&toc_links)
            .toc_links(&doc.toc_links)
            .footer(cluster.context.marketing_footer.to_string());



        Ok(ResponseOk(
            layout.render(crate::templates::Article { content: html }),
            layout.render(crate::templates::Article { content:doc.html }),
        ))
    }
 }
    cluster: &Cluster,
    origin: &Origin<'_>,
 ) -> Result<ResponseOk, Status> {
    BLOG.get_content(path, cluster, origin).await
    let doc = BLOG.get_content(path.clone(), "blog", origin).await;
    BLOG.render(doc, cluster).await
 }

 #[get("/careers/<path..>", rank = 5)]
    cluster: &Cluster,
    origin: &Origin<'_>,
 ) -> Result<ResponseOk, Status> {
    CAREERS.get_content(path, cluster, origin).await
    let doc = CAREERS.get_content(path, "careers", origin).await;
    CAREERS.render(doc, cluster).await
 }

 #[get("/docs/<path..>", rank = 5)]
    cluster: &Cluster,
    origin: &Origin<'_>,
 ) -> Result<ResponseOk, Status> {
    DOCS.get_content(path, cluster, origin).await
    let doc = DOCS.get_content(path.clone(), "docs", origin).await;
    DOCS.render(doc, cluster).await
 }

 pub fn routes() -> Vec<Route> {
diff --git a/pgml-dashboard/src/components/layouts/head/mod.rs b/pgml-dashboard/src/components/layouts/head/mod.rs

 #[cfg(test)]
 mod head_tests {
    usecrate::templates::Head;
    usesuper::Head;

    #[test]
    fn new_head() {
        );
    }

    #[test]
    fn add_preload() {
        let mut head = Head::new();
        let mut preloads: Vec<String> = vec![];
        for i in 0..5 {
            preloads.push(format!("image/test_preload_{}.test", i).to_string());
        }
        for preload in preloads.clone() {
            head.add_preload(&preload);
        }
        assert!(head.preloads.eq(&preloads));
    }
 //#[test]
 //fn add_preload() {
 //    let mut head = Head::new();
 //    let mut preloads: Vec<String> = vec![];
 //    for i in 0..5 {
 //        preloads.push(format!("image/test_preload_{}.test", i).to_string());
 //    }
 //    for preload in preloads.clone() {
 //        head.add_preload(&preload);
 //    }
 //    assert!(head.preloads.eq(&preloads));
 //}

    #[test]
    fn add_title() {

 #[cfg(test)]
 mod default_head_template_test {
    use super::{DefaultHeadTemplate,Head};
    use super::Head;
    use sailfish::TemplateOnce;

    #[test]
    fn default() {
        let head =DefaultHeadTemplate::new(None);
        let head =Head::new();
        let rendered = head.render_once().unwrap();

        assert!(

    #[test]
    fn set_head() {
        let muthead_info = Head::new()
        let muthead = Head::new()
            .title("test title")
            .description("test description")
            .image("image/test_image.jpg");
 head_info.add_preload("image/test_preload.webp");
 // head.add_preload("image/test_preload.webp");

        let head = DefaultHeadTemplate::new(Some(head_info));
        let rendered = head.render_once().unwrap();
        assert!(
            rendered.contains("<title>test title – PostgresML</title>") &&
diff --git a/pgml-dashboard/src/main.rs b/pgml-dashboard/src/main.rs

    markdown::SearchIndex::build().await.unwrap();

    markdown::CmsParse::build().await;

    pgml_dashboard::migrate(guards::Cluster::default(None).pool())
        .await
        .unwrap();
diff --git a/pgml-dashboard/src/templates/docs.rs b/pgml-dashboard/src/templates/docs.rs
 use sailfish::TemplateOnce;
 use serde::{Deserialize, Serialize};

 use crate::utils::markdown::SearchResult;

 }

 /// Table of contents link.
 #[derive(Clone, Debug)]
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct TocLink {
    pub title: String,
    pub id: String,
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,3 +4,4 @@ search_index
		.DS_Store
		.DS_Store/
		node_modules
		cms/
Original file line number	Diff line number	Diff line change
Expand Up		@@ -19,12 +19,94 @@ use crate::{
		utils::config,
		};

		use serde::{Deserialize, Serialize};

		lazy_static! {
		static ref BLOG: Collection = Collection::new("Blog", true);
		static ref CAREERS: Collection = Collection::new("Careers", true);
		static ref DOCS: Collection = Collection::new("Docs", false);
		}

		#[derive(Debug, Serialize, Deserialize)]
		pub struct Document {
		/// The absolute path on disk
		pub path: PathBuf,
		pub description: Option<String>,
		pub image: Option<String>,
		pub title: String,
		pub toc_links: Vec<TocLink>,
		pub html: String,
		}

		impl Document {
		pub async fn from_path(path: &PathBuf) -> anyhow::Result<Document> {
		let contents = tokio::fs::read_to_string(&path).await?;

		let parts = contents.split("---").collect::<Vec<&str>>();

		let (description, contents) = if parts.len() > 1 {
		match YamlLoader::load_from_str(parts[1]) {
		Ok(meta) => {
		if meta.len() == 0 \|\| meta[0].as_hash().is_none() {
		(None, contents)
		} else {
		let description: Option<String> = match meta[0]["description"].is_badvalue()
		{
		true => None,
		false => Some(meta[0]["description"].as_str().unwrap().to_string()),
		};
		(description, parts[2..].join("---").to_string())
		}
		}
		Err(_) => (None, contents),
		}
		} else {
		(None, contents)
		};

		// Parse Markdown
		let arena = Arena::new();
		let spaced_contents = crate::utils::markdown::gitbook_preprocess(&contents);
		let root = parse_document(&arena, &spaced_contents, &crate::utils::markdown::options());

		// Title of the document is the first (and typically only) <h1>
		let title = crate::utils::markdown::get_title(root).unwrap();
		let toc_links = crate::utils::markdown::get_toc(root).unwrap();
		let image = crate::utils::markdown::get_image(root);
		crate::utils::markdown::wrap_tables(root, &arena).unwrap();

		// MkDocs, gitbook syntax support, e.g. tabs, notes, alerts, etc.
		crate::utils::markdown::mkdocs(root, &arena).unwrap();

		// Style headings like we like them
		let mut plugins = ComrakPlugins::default();
		let headings = crate::utils::markdown::MarkdownHeadings::new();
		plugins.render.heading_adapter = Some(&headings);
		plugins.render.codefence_syntax_highlighter =
		Some(&crate::utils::markdown::SyntaxHighlighter {});

		let mut html = vec![];
		format_html_with_plugins(
		root,
		&crate::utils::markdown::options(),
		&mut html,
		&plugins,
		)
		.unwrap();
		let html = String::from_utf8(html).unwrap();

		let document = Document {
		path: path.to_owned(),
		description,
		image,
		title,
		toc_links,
		html,
		};
		Ok(document)
		}
		}

		/// A Gitbook collection of documents
		#[derive(Default)]
		struct Collection {
Expand DownExpand Up		@@ -62,24 +144,24 @@ impl Collection {

		pub async fn get_asset(&self, path: &str) -> Option<NamedFile> {
		info!("get_asset: {} {path}", self.name);

		NamedFile::open(self.asset_dir.join(path)).await.ok()
		}

		pub async fn get_content(
		&self,
		mut path: PathBuf,
		cluster: &Cluster,
		doc_type: &str,
		origin: &Origin<'_>,
		) -> Result<ResponseOk, Status> {
		info!("get_content: {} \| {path:?}", self.name);

		) -> Document {
		if origin.path().ends_with("/") {
		path = path.join("README");
		}

		let path =self.root_dir.join(format!("{}.md", path.to_string_lossy()));
		let path =PathBuf::from(format!("cms/{}/{}.json",doc_type,path.display()));

		self.render(&path, cluster, self).await
		let content = std::fs::read_to_string(path).unwrap();
		serde_json::from_str(&content).unwrap()
		}

		/// Create an index of the Collection based on the SUMMARY.md from Gitbook.
Expand DownExpand Up		@@ -173,123 +255,51 @@ impl Collection {
		Ok(links)
		}

		async fn render<'a>(
		&self,
		path: &'a PathBuf,
		cluster: &Cluster,
		collection: &Collection,
		) -> Result<ResponseOk, Status> {
		// Read to string0
		let contents = match tokio::fs::read_to_string(&path).await {
		Ok(contents) => {
		info!("loading markdown file: '{:?}", path);
		contents
		}
		Err(err) => {
		warn!("Error parsing markdown file: '{:?}' {:?}", path, err);
		return Err(Status::NotFound);
		}
		};
		let parts = contents.split("---").collect::<Vec<&str>>();
		let (description, contents) = if parts.len() > 1 {
		match YamlLoader::load_from_str(parts[1]) {
		Ok(meta) => {
		if !meta.is_empty() {
		let meta = meta[0].clone();
		if meta.as_hash().is_none() {
		(None, contents.to_string())
		} else {
		let description: Option<String> = match meta["description"]
		.is_badvalue()
		{
		true => None,
		false => Some(meta["description"].as_str().unwrap().to_string()),
		};

		(description, parts[2..].join("---").to_string())
		}
		} else {
		(None, contents.to_string())
		}
		}
		Err(_) => (None, contents.to_string()),
		}
		} else {
		(None, contents.to_string())
		};

		// Parse Markdown
		let arena = Arena::new();
		let root = parse_document(&arena, &contents, &crate::utils::markdown::options());

		// Title of the document is the first (and typically only) <h1>
		let title = crate::utils::markdown::get_title(root).unwrap();
		let toc_links = crate::utils::markdown::get_toc(root).unwrap();
		let image = crate::utils::markdown::get_image(root);
		crate::utils::markdown::wrap_tables(root, &arena).unwrap();

		// MkDocs syntax support, e.g. tabs, notes, alerts, etc.
		crate::utils::markdown::mkdocs(root, &arena).unwrap();

		// Style headings like we like them
		let mut plugins = ComrakPlugins::default();
		let headings = crate::utils::markdown::MarkdownHeadings::new();
		plugins.render.heading_adapter = Some(&headings);
		plugins.render.codefence_syntax_highlighter =
		Some(&crate::utils::markdown::SyntaxHighlighter {});

		// Render
		let mut html = vec![];
		format_html_with_plugins(
		root,
		&crate::utils::markdown::options(),
		&mut html,
		&plugins,
		)
		.unwrap();
		let html = String::from_utf8(html).unwrap();

		// Handle navigation
		// TODO organize this functionality in the collection to cleanup
		let index: Vec<IndexLink> = self
		.index
		// Sets specified index as currently viewed.
		pub fn open_index(&self, path: PathBuf) -> Vec<IndexLink> {
		self.index
		.clone()
		.iter_mut()
		.map(\|nav_link\| {
		let mut nav_link = nav_link.clone();
		nav_link.should_open(path);
		nav_link.should_open(&path);
		nav_link
		})
		.collect();
		.collect()
		}

		async fn render<'a>(&self, doc: Document, cluster: &Cluster) -> Result<ResponseOk, Status> {
		let index = self.open_index(doc.path);

		let user = if cluster.context.user.is_anonymous() {
		None
		} else {
		Some(cluster.context.user.clone())
		};

		let mut layout = crate::templates::Layout::new(&title, Some(cluster));
		if let Some(image) = image {
		// translate relative url into absolute for head social sharing
		let parts = image.split(".gitbook/assets/").collect::<Vec<&str>>();
		let image_path = collection.url_root.join(".gitbook/assets").join(parts[1]);
		layout.image(config::asset_url(image_path.to_string_lossy()).as_ref());
		let mut layout = crate::templates::Layout::new(&doc.title, Some(cluster));
		if let Some(image) = doc.image {
		layout.image(&config::asset_url(image.into()));
		}
		if let Some(description) = &description {
		if let Some(description) = &doc.description {
		layout.description(description);
		}
		if let Some(user) = &user {
		layout.user(user);
		}



		let layout = layout
		.nav_title(&self.name)
		.nav_links(&index)
		.toc_links(&toc_links)
		.toc_links(&doc.toc_links)
		.footer(cluster.context.marketing_footer.to_string());



		Ok(ResponseOk(
		layout.render(crate::templates::Article { content: html }),
		layout.render(crate::templates::Article { content:doc.html }),
		))
		}
		}
Expand DownExpand Up		@@ -328,7 +338,8 @@ async fn get_blog(
		cluster: &Cluster,
		origin: &Origin<'_>,
		) -> Result<ResponseOk, Status> {
		BLOG.get_content(path, cluster, origin).await
		let doc = BLOG.get_content(path.clone(), "blog", origin).await;
		BLOG.render(doc, cluster).await
		}

		#[get("/careers/<path..>", rank = 5)]
Expand All		@@ -337,7 +348,8 @@ async fn get_careers(
		cluster: &Cluster,
		origin: &Origin<'_>,
		) -> Result<ResponseOk, Status> {
		CAREERS.get_content(path, cluster, origin).await
		let doc = CAREERS.get_content(path, "careers", origin).await;
		CAREERS.render(doc, cluster).await
		}

		#[get("/docs/<path..>", rank = 5)]
Expand All		@@ -346,7 +358,8 @@ async fn get_docs(
		cluster: &Cluster,
		origin: &Origin<'_>,
		) -> Result<ResponseOk, Status> {
		DOCS.get_content(path, cluster, origin).await
		let doc = DOCS.get_content(path.clone(), "docs", origin).await;
		DOCS.render(doc, cluster).await
		}

		pub fn routes() -> Vec<Route> {
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -50,7 +50,7 @@ component!(Head);

		#[cfg(test)]
		mod head_tests {
		usecrate::templates::Head;
		usesuper::Head;

		#[test]
		fn new_head() {
Expand All		@@ -61,18 +61,18 @@ mod head_tests {
		);
		}

		#[test]
		fn add_preload() {
		let mut head = Head::new();
		let mut preloads: Vec<String> = vec![];
		for i in 0..5 {
		preloads.push(format!("image/test_preload_{}.test", i).to_string());
		}
		for preload in preloads.clone() {
		head.add_preload(&preload);
		}
		assert!(head.preloads.eq(&preloads));
		}
		//#[test]
		//fn add_preload() {
		// let mut head = Head::new();
		// let mut preloads: Vec<String> = vec![];
		// for i in 0..5 {
		// preloads.push(format!("image/test_preload_{}.test", i).to_string());
		// }
		// for preload in preloads.clone() {
		// head.add_preload(&preload);
		// }
		// assert!(head.preloads.eq(&preloads));
		//}

		#[test]
		fn add_title() {
Expand DownExpand Up		@@ -101,12 +101,12 @@ mod head_tests {

		#[cfg(test)]
		mod default_head_template_test {
		use super::{DefaultHeadTemplate,Head};
		use super::Head;
		use sailfish::TemplateOnce;

		#[test]
		fn default() {
		let head =DefaultHeadTemplate::new(None);
		let head =Head::new();
		let rendered = head.render_once().unwrap();

		assert!(
Expand All		@@ -120,13 +120,12 @@ mod default_head_template_test {

		#[test]
		fn set_head() {
		let muthead_info = Head::new()
		let muthead = Head::new()
		.title("test title")
		.description("test description")
		.image("image/test_image.jpg");
		head_info.add_preload("image/test_preload.webp");
		// head.add_preload("image/test_preload.webp");

		let head = DefaultHeadTemplate::new(Some(head_info));
		let rendered = head.render_once().unwrap();
		assert!(
		rendered.contains("<title>test title – PostgresML</title>") &&
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -100,6 +100,8 @@ async fn main() {

		markdown::SearchIndex::build().await.unwrap();

		markdown::CmsParse::build().await;

		pgml_dashboard::migrate(guards::Cluster::default(None).pool())
		.await
		.unwrap();
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,4 +1,5 @@
		use sailfish::TemplateOnce;
		use serde::{Deserialize, Serialize};

		use crate::utils::markdown::SearchResult;

Expand All		@@ -11,7 +12,7 @@ pub struct Search {
		}

		/// Table of contents link.
		#[derive(Clone, Debug)]
		#[derive(Clone, Debug, Serialize, Deserialize)]
		pub struct TocLink {
		pub title: String,
		pub id: String,
Expand Down