Oct 12, 2025 · Oct 12, 2025 · Oct 12, 2025 · Oct 13, 2025 · Oct 13, 2025 · Oct 13, 2025
diff --git a/docs/EmbedLang-Perf-TODO.md b/docs/EmbedLang-Perf-TODO.md
 # EmbedLang Performance TODO

 This document tracks incremental improvements to EmbedLang performance in Rewatch. Each step should land with tests passing and without changing user-facing semantics.

 Suggested order of changes (from EmbedLang design):

 1) Single index read + tag→generator map
 - Read `*.embeds.json` once per module and reuse the parsed structure for both planning and processing.
 - Build a per-package `tag -> generator` map once and reuse for O(1) lookups.

 2) Global scheduling + batch parse
 - Replace per‑module pools/loops with a single global work queue across all modules.
 - Accumulate generated files and parse them via the standard parallel AST generation (avoid per‑file `generate_ast`).

 3) Batch‑first (one‑shot) protocol
 - Group requests per `generator.id` and send in batches to reduce process overhead while keeping one‑shot execution.

 4) Daemon mode scaffolding
 - Add optional persistent per‑generator processes with a simple stdio protocol and deterministic queues.

 5) Watch maps + cleanup reductions
 - Maintain in‑memory maps for `extraSource -> tags -> modules` to avoid disk scans.
 - Reduce O(outDir) scans in cleanup by grouping or per‑module manifests.

diff --git a/rewatch/src/build.rs b/rewatch/src/build.rs
    let timing_embeds = Instant::now();
    {
        let mut embeds_had_failure = false;
        // Collect work items first to avoid borrow conflicts
        let mut work: Vec<(String, String, std::path::PathBuf, std::path::PathBuf)> = Vec::new();
        // Collect work items first to avoid borrow conflicts. Preload embed indexes once.
        // Store the cloned Package for parallel work to avoid shared map lookups.
        let mut work: Vec<(
            String,                          // module_name
            crate::build::packages::Package, // package
            std::path::PathBuf,              // impl_rel
            std::path::PathBuf,              // ast_rel
            Option<crate::build::embeds::EmbedIndexFile>, // preloaded index
        )> = Vec::new();
        for (module_name, package_name) in build_state.module_name_package_pairs() {
            if let Some(module) = build_state.build_state.modules.get(&module_name)
                && let SourceType::SourceFile(source_file) = &module.source_type
            {
                let ast_path_rel = helpers::get_ast_path(&source_file.implementation.path);
                // Try to preload the embeds index if present
                let idx_rel = {
                    let stem = ast_path_rel
                        .file_stem()
                        .unwrap_or_default()
                        .to_string_lossy()
                        .to_string();
                    ast_path_rel
                        .parent()
                        .unwrap_or_else(|| std::path::Path::new(""))
                        .join(format!("{stem}.embeds.json"))
                };
                let package_ref = build_state
                    .build_state
                    .packages
                    .get(&package_name)
                    .expect("Package not found")
                    .clone();
                let idx_abs = package_ref.get_build_path().join(&idx_rel);
                let preloaded_index = if idx_abs.exists() {
                    crate::build::embeds::read_index(&idx_abs).ok()
                } else {
                    None
                };

                work.push((
                    module_name.clone(),
 package_name.clone(),
 package_ref,
                    source_file.implementation.path.clone(),
                    ast_path_rel,
                    preloaded_index,
                ));
            }
        }
        let mut planned_invocations: u64 = 0;
        let mut planned_reused: u64 = 0;
        let mut per_module_invocations: Vec<(String, u64)> = Vec::new();
        for (module_name, package_name, _impl_rel, ast_rel) in &work {
            let package_ref = build_state
                .build_state
                .packages
                .get(package_name)
                .expect("Package not found");
            if let Ok((inv, reused)) = embeds::count_planned_invocations(build_state, package_ref, ast_rel) {
        for (module_name, package_ref, _impl_rel, ast_rel, preloaded_index) in &work {
            let (inv, reused) = if let Some(ix) = preloaded_index {
                embeds::count_planned_invocations_from_index(
                    package_ref,
                    package_ref
                        .config
                        .get_effective_embeds_config(&build_state.project_context)
                        .expect("embeds config present when index exists"),
                    ix,
                )
                .unwrap_or_default()
            } else {
                embeds::count_planned_invocations(build_state, package_ref, ast_rel).unwrap_or_default()
            };
            if inv > 0 || reused > 0 {
                planned_invocations += inv as u64;
                planned_reused += reused as u64;
                per_module_invocations.push((module_name.clone(), inv as u64));
            }
            per_module_invocations.push((module_name.clone(), inv as u64));
        }

        // Progress bar for generator invocations (non-verbose)
            ProgressBar::hidden()
        };

        for (module_name, package_name, impl_rel, ast_rel) in &work {
            let result = {
                let package_ref = build_state
                    .build_state
                    .packages
                    .get(package_name)
                    .expect("Package not found")
                    .clone();
                embeds::process_module_embeds(build_state, package_ref, impl_rel, ast_rel)
            };
        // Process modules in parallel (global scheduling across modules) using preloaded indexes.
        use rayon::prelude::*;
        let results: Vec<(String, crate::build::packages::Package, anyhow::Result<Vec<embeds::GeneratedModuleInfo>>)> =
            work
                .into_par_iter()
                .map(|(module_name, package, _impl_rel, ast_rel, preloaded_index)| {
                    let ix_opt = match preloaded_index {
                        Some(ix) => Some(ix),
                        None => {
                            // Attempt to read index if present
                            let stem = ast_rel
                                .file_stem()
                                .unwrap_or_default()
                                .to_string_lossy()
                                .to_string();
                            let idx_rel = ast_rel
                                .parent()
                                .unwrap_or_else(|| std::path::Path::new(""))
                                .join(format!("{stem}.embeds.json"));
                            let idx_abs = package.get_build_path().join(&idx_rel);
                            if idx_abs.exists() {
                                crate::build::embeds::read_index(&idx_abs).ok()
                            } else {
                                None
                            }
                        }
                    };
                    let res = match ix_opt {
                        Some(ix) => embeds::process_module_embeds_with_index(
                            &build_state.project_context,
                            package.clone(),
                            &ast_rel,
                            &ix,
                        ),
                        None => {
                            // No index; perform cleanup only
                            embeds::cleanup_stale_generated_for_module(&package, &ast_rel, &[])
                                .map(|_| Vec::new())
                        }
                    };
                    (module_name, package, res)
                })
                .collect();

        // Merge results sequentially: register generated modules and update progress
        let mut any_generated = false;
        for (module_name, package, result) in results {
            match result {
                Ok(generated) => {
                    if !generated.is_empty() {
                        {
                            let package_ref = build_state
                                .build_state
                                .packages
                                .get(package_name)
                                .expect("Package not found")
                                .clone();
                            embeds::add_generated_modules_to_state(build_state, package_ref, &generated);
                        }
                        for g in generated {
                            let _ = parse::generate_ast(
                                build_state
                                    .build_state
                                    .packages
                                    .get(package_name)
                                    .expect("Package not found")
                                    .clone(),
                                &g.rel_path,
                                &build_state.build_state,
                                build_state.get_warn_error_override(),
                            );
                        }
                    }
                    if let Some((_, inv)) = per_module_invocations.iter().find(|(m, _)| m == module_name)
                        && *inv > 0
                    {
                        pb_embeds.inc(*inv);
                        embeds::add_generated_modules_to_state(build_state, package, &generated);
                        any_generated = true;
                    }
                }
                Err(e) => {
                    log::error!("Embed processing failed for {module_name}: {e}");
                    embeds_had_failure = true;
                    if let Some((_, inv)) = per_module_invocations.iter().find(|(m, _)| m == module_name)
                        && *inv > 0
                    {
                        pb_embeds.inc(*inv);
                    }
                }
            }
            if let Some((_, inv)) = per_module_invocations.iter().find(|(m, _)| m == &module_name)
                && *inv > 0
            {
                pb_embeds.inc(*inv);
            }
        }

        // Batch parse all generated modules in one pass for better throughput
        if any_generated {
            let _ = parse::generate_asts(build_state, || {});
        }

        if planned_invocations > 0 {
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,24 @@
		# EmbedLang Performance TODO

		This document tracks incremental improvements to EmbedLang performance in Rewatch. Each step should land with tests passing and without changing user-facing semantics.

		Suggested order of changes (from EmbedLang design):

		1) Single index read + tag→generator map
		- Read `*.embeds.json` once per module and reuse the parsed structure for both planning and processing.
		- Build a per-package `tag -> generator` map once and reuse for O(1) lookups.

		2) Global scheduling + batch parse
		- Replace per‑module pools/loops with a single global work queue across all modules.
		- Accumulate generated files and parse them via the standard parallel AST generation (avoid per‑file `generate_ast`).

		3) Batch‑first (one‑shot) protocol
		- Group requests per `generator.id` and send in batches to reduce process overhead while keeping one‑shot execution.

		4) Daemon mode scaffolding
		- Add optional persistent per‑generator processes with a simple stdio protocol and deterministic queues.

		5) Watch maps + cleanup reductions
		- Maintain in‑memory maps for `extraSource -> tags -> modules` to avoid disk scans.
		- Reduce O(outDir) scans in cleanup by grouping or per‑module manifests.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -376,18 +376,51 @@ pub fn incremental_build(
		let timing_embeds = Instant::now();
		{
		let mut embeds_had_failure = false;
		// Collect work items first to avoid borrow conflicts
		let mut work: Vec<(String, String, std::path::PathBuf, std::path::PathBuf)> = Vec::new();
		// Collect work items first to avoid borrow conflicts. Preload embed indexes once.
		// Store the cloned Package for parallel work to avoid shared map lookups.
		let mut work: Vec<(
		String, // module_name
		crate::build::packages::Package, // package
		std::path::PathBuf, // impl_rel
		std::path::PathBuf, // ast_rel
		Option<crate::build::embeds::EmbedIndexFile>, // preloaded index
		)> = Vec::new();
		for (module_name, package_name) in build_state.module_name_package_pairs() {
		if let Some(module) = build_state.build_state.modules.get(&module_name)
		&& let SourceType::SourceFile(source_file) = &module.source_type
		{
		let ast_path_rel = helpers::get_ast_path(&source_file.implementation.path);
		// Try to preload the embeds index if present
		let idx_rel = {
		let stem = ast_path_rel
		.file_stem()
		.unwrap_or_default()
		.to_string_lossy()
		.to_string();
		ast_path_rel
		.parent()
		.unwrap_or_else(\|\| std::path::Path::new(""))
		.join(format!("{stem}.embeds.json"))
		};
		let package_ref = build_state
		.build_state
		.packages
		.get(&package_name)
		.expect("Package not found")
		.clone();
		let idx_abs = package_ref.get_build_path().join(&idx_rel);
		let preloaded_index = if idx_abs.exists() {
		crate::build::embeds::read_index(&idx_abs).ok()
		} else {
		None
		};

		work.push((
		module_name.clone(),
		package_name.clone(),
		package_ref,
		source_file.implementation.path.clone(),
		ast_path_rel,
		preloaded_index,
		));
		}
		}
Expand All		@@ -399,17 +432,25 @@ pub fn incremental_build(
		let mut planned_invocations: u64 = 0;
		let mut planned_reused: u64 = 0;
		let mut per_module_invocations: Vec<(String, u64)> = Vec::new();
		for (module_name, package_name, _impl_rel, ast_rel) in &work {
		let package_ref = build_state
		.build_state
		.packages
		.get(package_name)
		.expect("Package not found");
		if let Ok((inv, reused)) = embeds::count_planned_invocations(build_state, package_ref, ast_rel) {
		for (module_name, package_ref, _impl_rel, ast_rel, preloaded_index) in &work {
		let (inv, reused) = if let Some(ix) = preloaded_index {
		embeds::count_planned_invocations_from_index(
		package_ref,
		package_ref
		.config
		.get_effective_embeds_config(&build_state.project_context)
		.expect("embeds config present when index exists"),
		ix,
		)
		.unwrap_or_default()
		} else {
		embeds::count_planned_invocations(build_state, package_ref, ast_rel).unwrap_or_default()
		};
		if inv > 0 \|\| reused > 0 {
		planned_invocations += inv as u64;
		planned_reused += reused as u64;
		per_module_invocations.push((module_name.clone(), inv as u64));
		}
		per_module_invocations.push((module_name.clone(), inv as u64));
		}

		// Progress bar for generator invocations (non-verbose)
Expand All		@@ -428,58 +469,75 @@ pub fn incremental_build(
		ProgressBar::hidden()
		};

		for (module_name, package_name, impl_rel, ast_rel) in &work {
		let result = {
		let package_ref = build_state
		.build_state
		.packages
		.get(package_name)
		.expect("Package not found")
		.clone();
		embeds::process_module_embeds(build_state, package_ref, impl_rel, ast_rel)
		};
		// Process modules in parallel (global scheduling across modules) using preloaded indexes.
		use rayon::prelude::*;
		let results: Vec<(String, crate::build::packages::Package, anyhow::Result<Vec<embeds::GeneratedModuleInfo>>)> =
		work
		.into_par_iter()
		.map(\|(module_name, package, _impl_rel, ast_rel, preloaded_index)\| {
		let ix_opt = match preloaded_index {
		Some(ix) => Some(ix),
		None => {
		// Attempt to read index if present
		let stem = ast_rel
		.file_stem()
		.unwrap_or_default()
		.to_string_lossy()
		.to_string();
		let idx_rel = ast_rel
		.parent()
		.unwrap_or_else(\|\| std::path::Path::new(""))
		.join(format!("{stem}.embeds.json"));
		let idx_abs = package.get_build_path().join(&idx_rel);
		if idx_abs.exists() {
		crate::build::embeds::read_index(&idx_abs).ok()
		} else {
		None
		}
		}
		};
		let res = match ix_opt {
		Some(ix) => embeds::process_module_embeds_with_index(
		&build_state.project_context,
		package.clone(),
		&ast_rel,
		&ix,
		),
		None => {
		// No index; perform cleanup only
		embeds::cleanup_stale_generated_for_module(&package, &ast_rel, &[])
		.map(\|_\| Vec::new())
		}
		};
		(module_name, package, res)
		})
		.collect();

		// Merge results sequentially: register generated modules and update progress
		let mut any_generated = false;
		for (module_name, package, result) in results {
		match result {
		Ok(generated) => {
		if !generated.is_empty() {
		{
		let package_ref = build_state
		.build_state
		.packages
		.get(package_name)
		.expect("Package not found")
		.clone();
		embeds::add_generated_modules_to_state(build_state, package_ref, &generated);
		}
		for g in generated {
		let _ = parse::generate_ast(
		build_state
		.build_state
		.packages
		.get(package_name)
		.expect("Package not found")
		.clone(),
		&g.rel_path,
		&build_state.build_state,
		build_state.get_warn_error_override(),
		);
		}
		}
		if let Some((_, inv)) = per_module_invocations.iter().find(\|(m, _)\| m == module_name)
		&& *inv > 0
		{
		pb_embeds.inc(*inv);
		embeds::add_generated_modules_to_state(build_state, package, &generated);
		any_generated = true;
		}
		}
		Err(e) => {
		log::error!("Embed processing failed for {module_name}: {e}");
		embeds_had_failure = true;
		if let Some((_, inv)) = per_module_invocations.iter().find(\|(m, _)\| m == module_name)
		&& *inv > 0
		{
		pb_embeds.inc(*inv);
		}
		}
		}
		if let Some((_, inv)) = per_module_invocations.iter().find(\|(m, _)\| m == &module_name)
		&& *inv > 0
		{
		pb_embeds.inc(*inv);
		}
		}

		// Batch parse all generated modules in one pass for better throughput
		if any_generated {
		let _ = parse::generate_asts(build_state, \|\| {});
		}

		if planned_invocations > 0 {
Expand Down