1 line
1.2 MiB
Plaintext
1 line
1.2 MiB
Plaintext
{"repo": "byteowlz/ingestr", "files": {"ingestr-cli/src/main.rs": {"b580df1d8ac145689a4d7496bf05d90608329fb0": {"t": 1785745866.8722813, "n": 149561, "h": "2e1c1f19886fb603", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, Instant, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse ocrs::{ImageSource, OcrEngine, OcrEngineParams};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse rten::Model as RtenModel;\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin)\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Batch convert all files with given extension in current directory (recursive)\n /// Example: --batch pptx converts all .pptx files to .md alongside sources\n #[arg(long, value_name = \"EXT\")]\n batch: Option<String>,\n /// Write output to a file or directory instead of stdout. When converting\n /// a directory and no output is specified, writes .md files alongside sources.\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Write output files alongside source files (same directory) when processing\n /// directories. This is the default when no --output is specified.\n #[arg(long)]\n in_place: bool,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"ocrs\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n Tesseract,\n #[default]\n Ocrs,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Ocrs => write!(f, \"ocrs\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n // lopdf can emit repetitive \"corrupt deflate stream\" warnings for encrypted/corrupt PDFs.\n // We handle these cases explicitly and return user-friendly errors, so hide this crate-level noise.\n builder.filter_module(\"lopdf\", LevelFilter::Error);\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, ocrs, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Ocrs,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Ocrs,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Check for encrypted PDFs before passing to markitdown to avoid panics\n if is_pdf_extension(&extension) {\n match is_pdf_encrypted(input) {\n Ok(true) => {\n // Encrypted PDF - try OCR if available, otherwise return error\n if self.ocr_enabled {\n if let Ok(ocr_result) = self.process_with_ocr(input) {\n if !ocr_result.text_content.trim().is_empty() {\n return Ok(ocr_result);\n }\n }\n }\n bail!(\n \"PDF is encrypted/password-protected and cannot be converted. \\\n Consider using --vlm flag to process it via a vision model, \\\n or --ocr flag to extract text via OCR.\"\n );\n }\n Ok(false) => {} // Not encrypted, proceed with markitdown\n Err(e) => {\n warn!(\"Could not check PDF encryption status: {}, attempting conversion anyway\", e);\n }\n }\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string())\n .trim_end_matches('/')\n .to_string();\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\n/// Check if a PDF file is encrypted/password-protected.\n/// Returns true if encrypted, false if not, or an error if the file cannot be read.\nfn is_pdf_encrypted(path: &Path) -> Result<bool> {\n use lopdf::{Document, Object};\n\n let doc = Document::load(path)\n .with_context(|| format!(\"failed to load PDF: {}\", path.display()))?;\n\n // Check the Encrypt dictionary in trailer\n if let Ok(encrypt) = doc.trailer.get(b\"Encrypt\") {\n if *encrypt != Object::Null {\n return Ok(true);\n }\n }\n\n Ok(false)\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096,\n // Disable thinking/reasoning for models that support it (Qwen3.5, etc.)\n // This puts the full response in \"content\" instead of \"reasoning\"\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n let message = &response_json[\"choices\"][0][\"message\"];\n\n // Try content first, then reasoning (for thinking models like Qwen3.5)\n let text = message[\"content\"]\n .as_str()\n .filter(|s| !s.is_empty())\n .or_else(|| message[\"reasoning\"].as_str())\n .or_else(|| message[\"reasoning_content\"].as_str());\n\n text.map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"no content in VLM response: {}\", response_json))\n}\n\nconst OCRS_DETECTION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten\";\nconst OCRS_RECOGNITION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten\";\n\nfn ocrs_cache_dir() -> Result<PathBuf> {\n let base_cache = std::env::var(\"XDG_CACHE_HOME\")\n .ok()\n .filter(|v| !v.trim().is_empty())\n .map(PathBuf::from)\n .or_else(dirs::cache_dir)\n .unwrap_or_else(|| {\n std::env::var(\"HOME\")\n .map(PathBuf::from)\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\".cache\")\n });\n\n let cache_dir = base_cache.join(\"ocrs\");\n fs::create_dir_all(&cache_dir)\n .with_context(|| format!(\"creating OCRS cache dir {}\", cache_dir.display()))?;\n Ok(cache_dir)\n}\n\nfn human_bytes(bytes: u64) -> String {\n const KB: f64 = 1024.0;\n const MB: f64 = KB * 1024.0;\n const GB: f64 = MB * 1024.0;\n\n let b = bytes as f64;\n if b >= GB {\n format!(\"{:.2} GB\", b / GB)\n } else if b >= MB {\n format!(\"{:.2} MB\", b / MB)\n } else if b >= KB {\n format!(\"{:.1} KB\", b / KB)\n } else {\n format!(\"{} B\", bytes)\n }\n}\n\nfn download_to_cache(url: &str, filename: &str) -> Result<PathBuf> {\n let path = ocrs_cache_dir()?.join(filename);\n if path.exists() {\n eprintln!(\"OCRS model cached: {}\", path.display());\n return Ok(path);\n }\n\n info!(\"downloading OCRS model from {}\", url);\n let mut response = reqwest::blocking::get(url)\n .with_context(|| format!(\"downloading model from {}\", url))?\n .error_for_status()\n .with_context(|| format!(\"failed to download model from {}\", url))?;\n\n let total = response.content_length();\n eprintln!(\n \"Downloading OCRS model {}{}\",\n filename,\n total\n .map(|t| format!(\" ({})\", human_bytes(t)))\n .unwrap_or_default()\n );\n\n let mut file = fs::File::create(&path)\n .with_context(|| format!(\"creating model file {}\", path.display()))?;\n let mut downloaded: u64 = 0;\n let mut buffer = vec![0_u8; 64 * 1024];\n let mut last_print = Instant::now();\n let started = Instant::now();\n\n loop {\n let n = response\n .read(&mut buffer)\n .with_context(|| format!(\"reading model bytes from {}\", url))?;\n if n == 0 {\n break;\n }\n\n file.write_all(&buffer[..n])\n .with_context(|| format!(\"writing model to {}\", path.display()))?;\n downloaded += n as u64;\n\n if last_print.elapsed() >= Duration::from_millis(200) {\n match total {\n Some(t) if t > 0 => {\n let pct = (downloaded as f64 / t as f64) * 100.0;\n eprint!(\n \"\\r -> {} / {} ({:.1}%)\",\n human_bytes(downloaded),\n human_bytes(t),\n pct\n );\n }\n _ => {\n eprint!(\"\\r -> downloaded {}\", human_bytes(downloaded));\n }\n }\n let _ = io::stderr().flush();\n last_print = Instant::now();\n }\n }\n\n let elapsed = started.elapsed().as_secs_f32();\n eprintln!(\n \"\\r -> done: {} in {:.1}s \",\n human_bytes(downloaded),\n elapsed\n );\n\n Ok(path)\n}\n\nfn run_ocrs_on_image(input: &Path, engine: &OcrEngine, show_progress: bool) -> Result<String> {\n let started = Instant::now();\n\n if show_progress {\n eprintln!(\"OCRS: loading image {}\", input.display());\n }\n let image = image::open(input)\n .with_context(|| format!(\"reading image for OCRS: {}\", input.display()))?\n .into_rgb8();\n\n let image_source = ImageSource::from_bytes(image.as_raw(), image.dimensions())\n .context(\"creating OCRS image source\")?;\n\n if show_progress {\n eprintln!(\"OCRS: preparing input tensor\");\n }\n let ocr_input = engine.prepare_input(image_source).context(\"preparing OCRS input\")?;\n\n if show_progress {\n eprintln!(\"OCRS: detecting words\");\n }\n let word_rects = engine.detect_words(&ocr_input).context(\"OCRS word detection failed\")?;\n\n if show_progress {\n eprintln!(\"OCRS: grouping into lines\");\n }\n let line_rects = engine.find_text_lines(&ocr_input, &word_rects);\n\n if show_progress {\n eprintln!(\"OCRS: recognizing text\");\n }\n let line_texts = engine\n .recognize_text(&ocr_input, &line_rects)\n .context(\"OCRS text recognition failed\")?;\n\n let text = line_texts\n .iter()\n .flatten()\n .map(|line| line.to_string())\n .filter(|line| !line.trim().is_empty())\n .collect::<Vec<_>>()\n .join(\"\\n\");\n\n if show_progress {\n eprintln!(\n \"OCRS: done in {:.2}s ({} words, {} lines)\",\n started.elapsed().as_secs_f32(),\n word_rects.len(),\n line_rects.len()\n );\n }\n\n Ok(text)\n}\n\nfn run_ocrs_on_pdf(input: &Path, engine: &OcrEngine) -> Result<String> {\n let temp_dir = std::env::temp_dir().join(format!(\"ingestr-ocrs-pdf-{}\", std::process::id()));\n if temp_dir.exists() {\n let _ = fs::remove_dir_all(&temp_dir);\n }\n fs::create_dir_all(&temp_dir)\n .with_context(|| format!(\"creating temp dir {}\", temp_dir.display()))?;\n\n let result = (|| -> Result<String> {\n eprint!(\"OCRS: rendering PDF pages...\");\n let render_started = Instant::now();\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total = page_images.len();\n eprintln!(\" {} pages in {:.1}s\", total, render_started.elapsed().as_secs_f32());\n\n let mut sections = Vec::with_capacity(total);\n let pipeline_started = Instant::now();\n\n for (idx, page) in page_images.iter().enumerate() {\n let page_idx = idx + 1;\n let page_started = Instant::now();\n let page_text = run_ocrs_on_image(page, engine, false)\n .with_context(|| format!(\"OCRS failed on PDF page {}\", page_idx))?;\n sections.push(format!(\"## Page {}\\n\\n{}\", page_idx, page_text));\n\n let done = page_idx;\n let avg = pipeline_started.elapsed().as_secs_f32() / done as f32;\n let remaining_pages = total.saturating_sub(done);\n let eta = avg * remaining_pages as f32;\n eprintln!(\n \"OCRS PDF: [{}/{}] page {} done ({:.2}s page, {:.2}s avg, ~{:.0}s remaining)\",\n done,\n total,\n page_idx,\n page_started.elapsed().as_secs_f32(),\n avg,\n eta\n );\n }\n\n eprintln!(\n \"OCRS PDF: completed {} pages in {:.1}s\",\n total,\n pipeline_started.elapsed().as_secs_f32()\n );\n\n Ok(sections.join(\"\\n\\n\"))\n })();\n\n let _ = fs::remove_dir_all(&temp_dir);\n result\n}\n\nfn run_ocrs(input: &Path, languages: &[String]) -> Result<String> {\n if !languages.is_empty() && languages.iter().all(|l| l != \"eng\") {\n warn!(\n \"ocrs backend currently works best for Latin/English; requested languages: {:?}\",\n languages\n );\n }\n\n let init_started = Instant::now();\n eprintln!(\"OCRS: preparing models\");\n let detection_model_path = download_to_cache(OCRS_DETECTION_MODEL_URL, \"text-detection.rten\")?;\n let recognition_model_path =\n download_to_cache(OCRS_RECOGNITION_MODEL_URL, \"text-recognition.rten\")?;\n\n eprintln!(\"OCRS: loading detection model {}\",&detection_model_path.display());\n let detection_model =\n RtenModel::load_file(&detection_model_path).context(\"loading OCRS detection model\")?;\n\n eprintln!(\"OCRS: loading recognition model {}\",&recognition_model_path.display());\n let recognition_model =\n RtenModel::load_file(&recognition_model_path).context(\"loading OCRS recognition model\")?;\n\n let engine = OcrEngine::new(OcrEngineParams {\n detection_model: Some(detection_model),\n recognition_model: Some(recognition_model),\n ..Default::default()\n })\n .context(\"initializing OCRS engine\")?;\n eprintln!(\"OCRS: models ready in {:.2}s\", init_started.elapsed().as_secs_f32());\n\n if input\n .extension()\n .and_then(|e| e.to_str())\n .is_some_and(|ext| ext.eq_ignore_ascii_case(\"pdf\"))\n {\n run_ocrs_on_pdf(input, &engine)\n } else {\n run_ocrs_on_image(input, &engine, true)\n }\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n\n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Ocrs => run_ocrs(input, languages),\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n\n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n // Handle batch conversion flag: --batch EXT\n if let Some(ref batch_ext) = cmd.batch {\n let ext = batch_ext.trim_start_matches('.'); // Allow \"pptx\" or \".pptx\"\n if ext.is_empty() {\n bail!(\"--batch requires a file extension, e.g., --batch pptx\");\n }\n let cmd = ConvertCommand {\n extensions: Some(vec![ext.to_string()]),\n recursive: true,\n in_place: true,\n input: Some(\".\".to_string()), // Use current directory as base\n ..cmd\n };\n return handle_convert_directory(ctx, &cmd, &settings, &env::current_dir()?);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Determine if we should write alongside source files\n let in_place = cmd.in_place || (output_dir.is_none() && !ctx.common.dry_run);\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n if !ctx.common.quiet {\n println!(\"Found {} files to convert\", total);\n }\n\n if ctx.common.dry_run {\n for file in &files {\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out.display().to_string()\n } else if let Some(ref out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n // Single-threaded mode gets interactive progress\n let show_progress = parallel == 1 && !ctx.common.quiet && !ctx.common.no_progress;\n let progress_counter = AtomicUsize::new(0);\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().enumerate().map(|(idx, file)| {\n // Show progress before processing\n if show_progress {\n let num = idx + 1;\n let file_name = file.file_name()\n .and_then(|n| n.to_str())\n .unwrap_or(\"unknown\");\n eprint!(\"\\r\\x1b[K[{}/{}] Converting {}...\", num, total, file_name);\n let _ = io::stderr().flush();\n }\n\n let result = process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n );\n\n // Show result indicator\n if show_progress {\n match &result {\n Ok(_) => eprint!(\" ✓\"),\n Err(_) => eprint!(\" ✗\"),\n }\n }\n\n result\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n // Add newline after progress indicator if used\n if show_progress {\n eprintln!();\n }\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n in_place: bool,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n // Determine output path: in-place (source dir), output dir, or stdout\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out\n } else if let Some(out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if in_place or output_dir is specified\n if in_place || output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Ocrs.to_string(), \"ocrs\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Ocrs);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "dbf3fcafec8218a777486b7b8427dcf763a3e828": {"t": 1785757399.8779972, "n": 149561, "h": "2e1c1f19886fb603", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, Instant, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse ocrs::{ImageSource, OcrEngine, OcrEngineParams};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse rten::Model as RtenModel;\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin)\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Batch convert all files with given extension in current directory (recursive)\n /// Example: --batch pptx converts all .pptx files to .md alongside sources\n #[arg(long, value_name = \"EXT\")]\n batch: Option<String>,\n /// Write output to a file or directory instead of stdout. When converting\n /// a directory and no output is specified, writes .md files alongside sources.\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Write output files alongside source files (same directory) when processing\n /// directories. This is the default when no --output is specified.\n #[arg(long)]\n in_place: bool,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"ocrs\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n Tesseract,\n #[default]\n Ocrs,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Ocrs => write!(f, \"ocrs\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n // lopdf can emit repetitive \"corrupt deflate stream\" warnings for encrypted/corrupt PDFs.\n // We handle these cases explicitly and return user-friendly errors, so hide this crate-level noise.\n builder.filter_module(\"lopdf\", LevelFilter::Error);\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, ocrs, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Ocrs,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Ocrs,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Check for encrypted PDFs before passing to markitdown to avoid panics\n if is_pdf_extension(&extension) {\n match is_pdf_encrypted(input) {\n Ok(true) => {\n // Encrypted PDF - try OCR if available, otherwise return error\n if self.ocr_enabled {\n if let Ok(ocr_result) = self.process_with_ocr(input) {\n if !ocr_result.text_content.trim().is_empty() {\n return Ok(ocr_result);\n }\n }\n }\n bail!(\n \"PDF is encrypted/password-protected and cannot be converted. \\\n Consider using --vlm flag to process it via a vision model, \\\n or --ocr flag to extract text via OCR.\"\n );\n }\n Ok(false) => {} // Not encrypted, proceed with markitdown\n Err(e) => {\n warn!(\"Could not check PDF encryption status: {}, attempting conversion anyway\", e);\n }\n }\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string())\n .trim_end_matches('/')\n .to_string();\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\n/// Check if a PDF file is encrypted/password-protected.\n/// Returns true if encrypted, false if not, or an error if the file cannot be read.\nfn is_pdf_encrypted(path: &Path) -> Result<bool> {\n use lopdf::{Document, Object};\n\n let doc = Document::load(path)\n .with_context(|| format!(\"failed to load PDF: {}\", path.display()))?;\n\n // Check the Encrypt dictionary in trailer\n if let Ok(encrypt) = doc.trailer.get(b\"Encrypt\") {\n if *encrypt != Object::Null {\n return Ok(true);\n }\n }\n\n Ok(false)\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096,\n // Disable thinking/reasoning for models that support it (Qwen3.5, etc.)\n // This puts the full response in \"content\" instead of \"reasoning\"\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n let message = &response_json[\"choices\"][0][\"message\"];\n\n // Try content first, then reasoning (for thinking models like Qwen3.5)\n let text = message[\"content\"]\n .as_str()\n .filter(|s| !s.is_empty())\n .or_else(|| message[\"reasoning\"].as_str())\n .or_else(|| message[\"reasoning_content\"].as_str());\n\n text.map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"no content in VLM response: {}\", response_json))\n}\n\nconst OCRS_DETECTION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten\";\nconst OCRS_RECOGNITION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten\";\n\nfn ocrs_cache_dir() -> Result<PathBuf> {\n let base_cache = std::env::var(\"XDG_CACHE_HOME\")\n .ok()\n .filter(|v| !v.trim().is_empty())\n .map(PathBuf::from)\n .or_else(dirs::cache_dir)\n .unwrap_or_else(|| {\n std::env::var(\"HOME\")\n .map(PathBuf::from)\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\".cache\")\n });\n\n let cache_dir = base_cache.join(\"ocrs\");\n fs::create_dir_all(&cache_dir)\n .with_context(|| format!(\"creating OCRS cache dir {}\", cache_dir.display()))?;\n Ok(cache_dir)\n}\n\nfn human_bytes(bytes: u64) -> String {\n const KB: f64 = 1024.0;\n const MB: f64 = KB * 1024.0;\n const GB: f64 = MB * 1024.0;\n\n let b = bytes as f64;\n if b >= GB {\n format!(\"{:.2} GB\", b / GB)\n } else if b >= MB {\n format!(\"{:.2} MB\", b / MB)\n } else if b >= KB {\n format!(\"{:.1} KB\", b / KB)\n } else {\n format!(\"{} B\", bytes)\n }\n}\n\nfn download_to_cache(url: &str, filename: &str) -> Result<PathBuf> {\n let path = ocrs_cache_dir()?.join(filename);\n if path.exists() {\n eprintln!(\"OCRS model cached: {}\", path.display());\n return Ok(path);\n }\n\n info!(\"downloading OCRS model from {}\", url);\n let mut response = reqwest::blocking::get(url)\n .with_context(|| format!(\"downloading model from {}\", url))?\n .error_for_status()\n .with_context(|| format!(\"failed to download model from {}\", url))?;\n\n let total = response.content_length();\n eprintln!(\n \"Downloading OCRS model {}{}\",\n filename,\n total\n .map(|t| format!(\" ({})\", human_bytes(t)))\n .unwrap_or_default()\n );\n\n let mut file = fs::File::create(&path)\n .with_context(|| format!(\"creating model file {}\", path.display()))?;\n let mut downloaded: u64 = 0;\n let mut buffer = vec![0_u8; 64 * 1024];\n let mut last_print = Instant::now();\n let started = Instant::now();\n\n loop {\n let n = response\n .read(&mut buffer)\n .with_context(|| format!(\"reading model bytes from {}\", url))?;\n if n == 0 {\n break;\n }\n\n file.write_all(&buffer[..n])\n .with_context(|| format!(\"writing model to {}\", path.display()))?;\n downloaded += n as u64;\n\n if last_print.elapsed() >= Duration::from_millis(200) {\n match total {\n Some(t) if t > 0 => {\n let pct = (downloaded as f64 / t as f64) * 100.0;\n eprint!(\n \"\\r -> {} / {} ({:.1}%)\",\n human_bytes(downloaded),\n human_bytes(t),\n pct\n );\n }\n _ => {\n eprint!(\"\\r -> downloaded {}\", human_bytes(downloaded));\n }\n }\n let _ = io::stderr().flush();\n last_print = Instant::now();\n }\n }\n\n let elapsed = started.elapsed().as_secs_f32();\n eprintln!(\n \"\\r -> done: {} in {:.1}s \",\n human_bytes(downloaded),\n elapsed\n );\n\n Ok(path)\n}\n\nfn run_ocrs_on_image(input: &Path, engine: &OcrEngine, show_progress: bool) -> Result<String> {\n let started = Instant::now();\n\n if show_progress {\n eprintln!(\"OCRS: loading image {}\", input.display());\n }\n let image = image::open(input)\n .with_context(|| format!(\"reading image for OCRS: {}\", input.display()))?\n .into_rgb8();\n\n let image_source = ImageSource::from_bytes(image.as_raw(), image.dimensions())\n .context(\"creating OCRS image source\")?;\n\n if show_progress {\n eprintln!(\"OCRS: preparing input tensor\");\n }\n let ocr_input = engine.prepare_input(image_source).context(\"preparing OCRS input\")?;\n\n if show_progress {\n eprintln!(\"OCRS: detecting words\");\n }\n let word_rects = engine.detect_words(&ocr_input).context(\"OCRS word detection failed\")?;\n\n if show_progress {\n eprintln!(\"OCRS: grouping into lines\");\n }\n let line_rects = engine.find_text_lines(&ocr_input, &word_rects);\n\n if show_progress {\n eprintln!(\"OCRS: recognizing text\");\n }\n let line_texts = engine\n .recognize_text(&ocr_input, &line_rects)\n .context(\"OCRS text recognition failed\")?;\n\n let text = line_texts\n .iter()\n .flatten()\n .map(|line| line.to_string())\n .filter(|line| !line.trim().is_empty())\n .collect::<Vec<_>>()\n .join(\"\\n\");\n\n if show_progress {\n eprintln!(\n \"OCRS: done in {:.2}s ({} words, {} lines)\",\n started.elapsed().as_secs_f32(),\n word_rects.len(),\n line_rects.len()\n );\n }\n\n Ok(text)\n}\n\nfn run_ocrs_on_pdf(input: &Path, engine: &OcrEngine) -> Result<String> {\n let temp_dir = std::env::temp_dir().join(format!(\"ingestr-ocrs-pdf-{}\", std::process::id()));\n if temp_dir.exists() {\n let _ = fs::remove_dir_all(&temp_dir);\n }\n fs::create_dir_all(&temp_dir)\n .with_context(|| format!(\"creating temp dir {}\", temp_dir.display()))?;\n\n let result = (|| -> Result<String> {\n eprint!(\"OCRS: rendering PDF pages...\");\n let render_started = Instant::now();\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total = page_images.len();\n eprintln!(\" {} pages in {:.1}s\", total, render_started.elapsed().as_secs_f32());\n\n let mut sections = Vec::with_capacity(total);\n let pipeline_started = Instant::now();\n\n for (idx, page) in page_images.iter().enumerate() {\n let page_idx = idx + 1;\n let page_started = Instant::now();\n let page_text = run_ocrs_on_image(page, engine, false)\n .with_context(|| format!(\"OCRS failed on PDF page {}\", page_idx))?;\n sections.push(format!(\"## Page {}\\n\\n{}\", page_idx, page_text));\n\n let done = page_idx;\n let avg = pipeline_started.elapsed().as_secs_f32() / done as f32;\n let remaining_pages = total.saturating_sub(done);\n let eta = avg * remaining_pages as f32;\n eprintln!(\n \"OCRS PDF: [{}/{}] page {} done ({:.2}s page, {:.2}s avg, ~{:.0}s remaining)\",\n done,\n total,\n page_idx,\n page_started.elapsed().as_secs_f32(),\n avg,\n eta\n );\n }\n\n eprintln!(\n \"OCRS PDF: completed {} pages in {:.1}s\",\n total,\n pipeline_started.elapsed().as_secs_f32()\n );\n\n Ok(sections.join(\"\\n\\n\"))\n })();\n\n let _ = fs::remove_dir_all(&temp_dir);\n result\n}\n\nfn run_ocrs(input: &Path, languages: &[String]) -> Result<String> {\n if !languages.is_empty() && languages.iter().all(|l| l != \"eng\") {\n warn!(\n \"ocrs backend currently works best for Latin/English; requested languages: {:?}\",\n languages\n );\n }\n\n let init_started = Instant::now();\n eprintln!(\"OCRS: preparing models\");\n let detection_model_path = download_to_cache(OCRS_DETECTION_MODEL_URL, \"text-detection.rten\")?;\n let recognition_model_path =\n download_to_cache(OCRS_RECOGNITION_MODEL_URL, \"text-recognition.rten\")?;\n\n eprintln!(\"OCRS: loading detection model {}\",&detection_model_path.display());\n let detection_model =\n RtenModel::load_file(&detection_model_path).context(\"loading OCRS detection model\")?;\n\n eprintln!(\"OCRS: loading recognition model {}\",&recognition_model_path.display());\n let recognition_model =\n RtenModel::load_file(&recognition_model_path).context(\"loading OCRS recognition model\")?;\n\n let engine = OcrEngine::new(OcrEngineParams {\n detection_model: Some(detection_model),\n recognition_model: Some(recognition_model),\n ..Default::default()\n })\n .context(\"initializing OCRS engine\")?;\n eprintln!(\"OCRS: models ready in {:.2}s\", init_started.elapsed().as_secs_f32());\n\n if input\n .extension()\n .and_then(|e| e.to_str())\n .is_some_and(|ext| ext.eq_ignore_ascii_case(\"pdf\"))\n {\n run_ocrs_on_pdf(input, &engine)\n } else {\n run_ocrs_on_image(input, &engine, true)\n }\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n\n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Ocrs => run_ocrs(input, languages),\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n\n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n // Handle batch conversion flag: --batch EXT\n if let Some(ref batch_ext) = cmd.batch {\n let ext = batch_ext.trim_start_matches('.'); // Allow \"pptx\" or \".pptx\"\n if ext.is_empty() {\n bail!(\"--batch requires a file extension, e.g., --batch pptx\");\n }\n let cmd = ConvertCommand {\n extensions: Some(vec![ext.to_string()]),\n recursive: true,\n in_place: true,\n input: Some(\".\".to_string()), // Use current directory as base\n ..cmd\n };\n return handle_convert_directory(ctx, &cmd, &settings, &env::current_dir()?);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Determine if we should write alongside source files\n let in_place = cmd.in_place || (output_dir.is_none() && !ctx.common.dry_run);\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n if !ctx.common.quiet {\n println!(\"Found {} files to convert\", total);\n }\n\n if ctx.common.dry_run {\n for file in &files {\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out.display().to_string()\n } else if let Some(ref out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n // Single-threaded mode gets interactive progress\n let show_progress = parallel == 1 && !ctx.common.quiet && !ctx.common.no_progress;\n let progress_counter = AtomicUsize::new(0);\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().enumerate().map(|(idx, file)| {\n // Show progress before processing\n if show_progress {\n let num = idx + 1;\n let file_name = file.file_name()\n .and_then(|n| n.to_str())\n .unwrap_or(\"unknown\");\n eprint!(\"\\r\\x1b[K[{}/{}] Converting {}...\", num, total, file_name);\n let _ = io::stderr().flush();\n }\n\n let result = process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n );\n\n // Show result indicator\n if show_progress {\n match &result {\n Ok(_) => eprint!(\" ✓\"),\n Err(_) => eprint!(\" ✗\"),\n }\n }\n\n result\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n // Add newline after progress indicator if used\n if show_progress {\n eprintln!();\n }\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n in_place: bool,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n // Determine output path: in-place (source dir), output dir, or stdout\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out\n } else if let Some(out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if in_place or output_dir is specified\n if in_place || output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Ocrs.to_string(), \"ocrs\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Ocrs);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "43d8d762d467771e656d82ed3e9a61df695d8100": {"t": 1785757401.515679, "n": 149581, "h": "b0a82923c66c3b8e", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, Instant, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse ocrs::{ImageSource, OcrEngine, OcrEngineParams};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse rten::Model as RtenModel;\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin).\n /// Use \":EXT\" pattern (e.g., \":pptx\", \":pdf\", \":docx\") to batch convert all\n /// files with that extension recursively in the current directory.\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Write output to a file or directory instead of stdout. When converting\n /// a directory and no output is specified, writes .md files alongside sources.\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Write output files alongside source files (same directory) when processing\n /// directories. This is the default when no --output is specified.\n #[arg(long)]\n in_place: bool,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"ocrs\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n Tesseract,\n #[default]\n Ocrs,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Ocrs => write!(f, \"ocrs\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n // lopdf can emit repetitive \"corrupt deflate stream\" warnings for encrypted/corrupt PDFs.\n // We handle these cases explicitly and return user-friendly errors, so hide this crate-level noise.\n builder.filter_module(\"lopdf\", LevelFilter::Error);\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, ocrs, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Ocrs,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Ocrs,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Check for encrypted PDFs before passing to markitdown to avoid panics\n if is_pdf_extension(&extension) {\n match is_pdf_encrypted(input) {\n Ok(true) => {\n // Encrypted PDF - try OCR if available, otherwise return error\n if self.ocr_enabled {\n if let Ok(ocr_result) = self.process_with_ocr(input) {\n if !ocr_result.text_content.trim().is_empty() {\n return Ok(ocr_result);\n }\n }\n }\n bail!(\n \"PDF is encrypted/password-protected and cannot be converted. \\\n Consider using --vlm flag to process it via a vision model, \\\n or --ocr flag to extract text via OCR.\"\n );\n }\n Ok(false) => {} // Not encrypted, proceed with markitdown\n Err(e) => {\n warn!(\"Could not check PDF encryption status: {}, attempting conversion anyway\", e);\n }\n }\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string())\n .trim_end_matches('/')\n .to_string();\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\n/// Check if a PDF file is encrypted/password-protected.\n/// Returns true if encrypted, false if not, or an error if the file cannot be read.\nfn is_pdf_encrypted(path: &Path) -> Result<bool> {\n use lopdf::{Document, Object};\n\n let doc = Document::load(path)\n .with_context(|| format!(\"failed to load PDF: {}\", path.display()))?;\n\n // Check the Encrypt dictionary in trailer\n if let Ok(encrypt) = doc.trailer.get(b\"Encrypt\") {\n if *encrypt != Object::Null {\n return Ok(true);\n }\n }\n\n Ok(false)\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096,\n // Disable thinking/reasoning for models that support it (Qwen3.5, etc.)\n // This puts the full response in \"content\" instead of \"reasoning\"\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n let message = &response_json[\"choices\"][0][\"message\"];\n\n // Try content first, then reasoning (for thinking models like Qwen3.5)\n let text = message[\"content\"]\n .as_str()\n .filter(|s| !s.is_empty())\n .or_else(|| message[\"reasoning\"].as_str())\n .or_else(|| message[\"reasoning_content\"].as_str());\n\n text.map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"no content in VLM response: {}\", response_json))\n}\n\nconst OCRS_DETECTION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten\";\nconst OCRS_RECOGNITION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten\";\n\nfn ocrs_cache_dir() -> Result<PathBuf> {\n let base_cache = std::env::var(\"XDG_CACHE_HOME\")\n .ok()\n .filter(|v| !v.trim().is_empty())\n .map(PathBuf::from)\n .or_else(dirs::cache_dir)\n .unwrap_or_else(|| {\n std::env::var(\"HOME\")\n .map(PathBuf::from)\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\".cache\")\n });\n\n let cache_dir = base_cache.join(\"ocrs\");\n fs::create_dir_all(&cache_dir)\n .with_context(|| format!(\"creating OCRS cache dir {}\", cache_dir.display()))?;\n Ok(cache_dir)\n}\n\nfn human_bytes(bytes: u64) -> String {\n const KB: f64 = 1024.0;\n const MB: f64 = KB * 1024.0;\n const GB: f64 = MB * 1024.0;\n\n let b = bytes as f64;\n if b >= GB {\n format!(\"{:.2} GB\", b / GB)\n } else if b >= MB {\n format!(\"{:.2} MB\", b / MB)\n } else if b >= KB {\n format!(\"{:.1} KB\", b / KB)\n } else {\n format!(\"{} B\", bytes)\n }\n}\n\nfn download_to_cache(url: &str, filename: &str) -> Result<PathBuf> {\n let path = ocrs_cache_dir()?.join(filename);\n if path.exists() {\n eprintln!(\"OCRS model cached: {}\", path.display());\n return Ok(path);\n }\n\n info!(\"downloading OCRS model from {}\", url);\n let mut response = reqwest::blocking::get(url)\n .with_context(|| format!(\"downloading model from {}\", url))?\n .error_for_status()\n .with_context(|| format!(\"failed to download model from {}\", url))?;\n\n let total = response.content_length();\n eprintln!(\n \"Downloading OCRS model {}{}\",\n filename,\n total\n .map(|t| format!(\" ({})\", human_bytes(t)))\n .unwrap_or_default()\n );\n\n let mut file = fs::File::create(&path)\n .with_context(|| format!(\"creating model file {}\", path.display()))?;\n let mut downloaded: u64 = 0;\n let mut buffer = vec![0_u8; 64 * 1024];\n let mut last_print = Instant::now();\n let started = Instant::now();\n\n loop {\n let n = response\n .read(&mut buffer)\n .with_context(|| format!(\"reading model bytes from {}\", url))?;\n if n == 0 {\n break;\n }\n\n file.write_all(&buffer[..n])\n .with_context(|| format!(\"writing model to {}\", path.display()))?;\n downloaded += n as u64;\n\n if last_print.elapsed() >= Duration::from_millis(200) {\n match total {\n Some(t) if t > 0 => {\n let pct = (downloaded as f64 / t as f64) * 100.0;\n eprint!(\n \"\\r -> {} / {} ({:.1}%)\",\n human_bytes(downloaded),\n human_bytes(t),\n pct\n );\n }\n _ => {\n eprint!(\"\\r -> downloaded {}\", human_bytes(downloaded));\n }\n }\n let _ = io::stderr().flush();\n last_print = Instant::now();\n }\n }\n\n let elapsed = started.elapsed().as_secs_f32();\n eprintln!(\n \"\\r -> done: {} in {:.1}s \",\n human_bytes(downloaded),\n elapsed\n );\n\n Ok(path)\n}\n\nfn run_ocrs_on_image(input: &Path, engine: &OcrEngine, show_progress: bool) -> Result<String> {\n let started = Instant::now();\n\n if show_progress {\n eprintln!(\"OCRS: loading image {}\", input.display());\n }\n let image = image::open(input)\n .with_context(|| format!(\"reading image for OCRS: {}\", input.display()))?\n .into_rgb8();\n\n let image_source = ImageSource::from_bytes(image.as_raw(), image.dimensions())\n .context(\"creating OCRS image source\")?;\n\n if show_progress {\n eprintln!(\"OCRS: preparing input tensor\");\n }\n let ocr_input = engine.prepare_input(image_source).context(\"preparing OCRS input\")?;\n\n if show_progress {\n eprintln!(\"OCRS: detecting words\");\n }\n let word_rects = engine.detect_words(&ocr_input).context(\"OCRS word detection failed\")?;\n\n if show_progress {\n eprintln!(\"OCRS: grouping into lines\");\n }\n let line_rects = engine.find_text_lines(&ocr_input, &word_rects);\n\n if show_progress {\n eprintln!(\"OCRS: recognizing text\");\n }\n let line_texts = engine\n .recognize_text(&ocr_input, &line_rects)\n .context(\"OCRS text recognition failed\")?;\n\n let text = line_texts\n .iter()\n .flatten()\n .map(|line| line.to_string())\n .filter(|line| !line.trim().is_empty())\n .collect::<Vec<_>>()\n .join(\"\\n\");\n\n if show_progress {\n eprintln!(\n \"OCRS: done in {:.2}s ({} words, {} lines)\",\n started.elapsed().as_secs_f32(),\n word_rects.len(),\n line_rects.len()\n );\n }\n\n Ok(text)\n}\n\nfn run_ocrs_on_pdf(input: &Path, engine: &OcrEngine) -> Result<String> {\n let temp_dir = std::env::temp_dir().join(format!(\"ingestr-ocrs-pdf-{}\", std::process::id()));\n if temp_dir.exists() {\n let _ = fs::remove_dir_all(&temp_dir);\n }\n fs::create_dir_all(&temp_dir)\n .with_context(|| format!(\"creating temp dir {}\", temp_dir.display()))?;\n\n let result = (|| -> Result<String> {\n eprint!(\"OCRS: rendering PDF pages...\");\n let render_started = Instant::now();\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total = page_images.len();\n eprintln!(\" {} pages in {:.1}s\", total, render_started.elapsed().as_secs_f32());\n\n let mut sections = Vec::with_capacity(total);\n let pipeline_started = Instant::now();\n\n for (idx, page) in page_images.iter().enumerate() {\n let page_idx = idx + 1;\n let page_started = Instant::now();\n let page_text = run_ocrs_on_image(page, engine, false)\n .with_context(|| format!(\"OCRS failed on PDF page {}\", page_idx))?;\n sections.push(format!(\"## Page {}\\n\\n{}\", page_idx, page_text));\n\n let done = page_idx;\n let avg = pipeline_started.elapsed().as_secs_f32() / done as f32;\n let remaining_pages = total.saturating_sub(done);\n let eta = avg * remaining_pages as f32;\n eprintln!(\n \"OCRS PDF: [{}/{}] page {} done ({:.2}s page, {:.2}s avg, ~{:.0}s remaining)\",\n done,\n total,\n page_idx,\n page_started.elapsed().as_secs_f32(),\n avg,\n eta\n );\n }\n\n eprintln!(\n \"OCRS PDF: completed {} pages in {:.1}s\",\n total,\n pipeline_started.elapsed().as_secs_f32()\n );\n\n Ok(sections.join(\"\\n\\n\"))\n })();\n\n let _ = fs::remove_dir_all(&temp_dir);\n result\n}\n\nfn run_ocrs(input: &Path, languages: &[String]) -> Result<String> {\n if !languages.is_empty() && languages.iter().all(|l| l != \"eng\") {\n warn!(\n \"ocrs backend currently works best for Latin/English; requested languages: {:?}\",\n languages\n );\n }\n\n let init_started = Instant::now();\n eprintln!(\"OCRS: preparing models\");\n let detection_model_path = download_to_cache(OCRS_DETECTION_MODEL_URL, \"text-detection.rten\")?;\n let recognition_model_path =\n download_to_cache(OCRS_RECOGNITION_MODEL_URL, \"text-recognition.rten\")?;\n\n eprintln!(\"OCRS: loading detection model {}\",&detection_model_path.display());\n let detection_model =\n RtenModel::load_file(&detection_model_path).context(\"loading OCRS detection model\")?;\n\n eprintln!(\"OCRS: loading recognition model {}\",&recognition_model_path.display());\n let recognition_model =\n RtenModel::load_file(&recognition_model_path).context(\"loading OCRS recognition model\")?;\n\n let engine = OcrEngine::new(OcrEngineParams {\n detection_model: Some(detection_model),\n recognition_model: Some(recognition_model),\n ..Default::default()\n })\n .context(\"initializing OCRS engine\")?;\n eprintln!(\"OCRS: models ready in {:.2}s\", init_started.elapsed().as_secs_f32());\n\n if input\n .extension()\n .and_then(|e| e.to_str())\n .is_some_and(|ext| ext.eq_ignore_ascii_case(\"pdf\"))\n {\n run_ocrs_on_pdf(input, &engine)\n } else {\n run_ocrs_on_image(input, &engine, true)\n }\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n\n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Ocrs => run_ocrs(input, languages),\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n\n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // Special patterns for batch conversion:\n // \":EXT\" -> find all .EXT files in current directory recursively and convert in-place\n // Examples: \":pptx\" for PowerPoints, \":pdf\" for PDFs, \":docx\" for Word files, etc.\n if input_str.starts_with(':') {\n let ext = &input_str[1..]; // remove the leading ':'\n if ext.is_empty() {\n bail!(\"special pattern \":EXT\" requires an extension, e.g., \":pptx\", \":pdf\", \":docx\"\");\n }\n let cmd = ConvertCommand {\n extensions: Some(vec![ext.to_string()]),\n recursive: true,\n in_place: true,\n ..cmd\n };\n return handle_convert_directory(ctx, &cmd, &settings, &env::current_dir()?);\n }\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Determine if we should write alongside source files\n let in_place = cmd.in_place || (output_dir.is_none() && !ctx.common.dry_run);\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n if !ctx.common.quiet {\n println!(\"Found {} files to convert\", total);\n }\n\n if ctx.common.dry_run {\n for file in &files {\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out.display().to_string()\n } else if let Some(ref out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n // Single-threaded mode gets interactive progress\n let show_progress = parallel == 1 && !ctx.common.quiet && !ctx.common.no_progress;\n let progress_counter = AtomicUsize::new(0);\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().enumerate().map(|(idx, file)| {\n // Show progress before processing\n if show_progress {\n let num = idx + 1;\n let file_name = file.file_name()\n .and_then(|n| n.to_str())\n .unwrap_or(\"unknown\");\n eprint!(\"\\r\\x1b[K[{}/{}] Converting {}...\", num, total, file_name);\n let _ = io::stderr().flush();\n }\n\n let result = process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n );\n\n // Show result indicator\n if show_progress {\n match &result {\n Ok(_) => eprint!(\" ✓\"),\n Err(_) => eprint!(\" ✗\"),\n }\n }\n\n result\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n // Add newline after progress indicator if used\n if show_progress {\n eprintln!();\n }\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n in_place: bool,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n // Determine output path: in-place (source dir), output dir, or stdout\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out\n } else if let Some(out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if in_place or output_dir is specified\n if in_place || output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Ocrs.to_string(), \"ocrs\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Ocrs);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "15e75dadc79313e38d11bc90ee5d96a0f6b80311": {"t": 1785757403.218706, "n": 149403, "h": "aa6e922193fbaf12", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, Instant, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse ocrs::{ImageSource, OcrEngine, OcrEngineParams};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse rten::Model as RtenModel;\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin). Use \":pptx\" to convert all PowerPoint files in current directory recursively.\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Write output to a file or directory instead of stdout. When converting\n /// a directory and no output is specified, writes .md files alongside sources.\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Write output files alongside source files (same directory) when processing\n /// directories. This is the default when no --output is specified.\n #[arg(long)]\n in_place: bool,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"ocrs\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n Tesseract,\n #[default]\n Ocrs,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Ocrs => write!(f, \"ocrs\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n // lopdf can emit repetitive \"corrupt deflate stream\" warnings for encrypted/corrupt PDFs.\n // We handle these cases explicitly and return user-friendly errors, so hide this crate-level noise.\n builder.filter_module(\"lopdf\", LevelFilter::Error);\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, ocrs, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Ocrs,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Ocrs,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Check for encrypted PDFs before passing to markitdown to avoid panics\n if is_pdf_extension(&extension) {\n match is_pdf_encrypted(input) {\n Ok(true) => {\n // Encrypted PDF - try OCR if available, otherwise return error\n if self.ocr_enabled {\n if let Ok(ocr_result) = self.process_with_ocr(input) {\n if !ocr_result.text_content.trim().is_empty() {\n return Ok(ocr_result);\n }\n }\n }\n bail!(\n \"PDF is encrypted/password-protected and cannot be converted. \\\n Consider using --vlm flag to process it via a vision model, \\\n or --ocr flag to extract text via OCR.\"\n );\n }\n Ok(false) => {} // Not encrypted, proceed with markitdown\n Err(e) => {\n warn!(\"Could not check PDF encryption status: {}, attempting conversion anyway\", e);\n }\n }\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string())\n .trim_end_matches('/')\n .to_string();\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\n/// Check if a PDF file is encrypted/password-protected.\n/// Returns true if encrypted, false if not, or an error if the file cannot be read.\nfn is_pdf_encrypted(path: &Path) -> Result<bool> {\n use lopdf::{Document, Object};\n\n let doc = Document::load(path)\n .with_context(|| format!(\"failed to load PDF: {}\", path.display()))?;\n\n // Check the Encrypt dictionary in trailer\n if let Ok(encrypt) = doc.trailer.get(b\"Encrypt\") {\n if *encrypt != Object::Null {\n return Ok(true);\n }\n }\n\n Ok(false)\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096,\n // Disable thinking/reasoning for models that support it (Qwen3.5, etc.)\n // This puts the full response in \"content\" instead of \"reasoning\"\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n let message = &response_json[\"choices\"][0][\"message\"];\n\n // Try content first, then reasoning (for thinking models like Qwen3.5)\n let text = message[\"content\"]\n .as_str()\n .filter(|s| !s.is_empty())\n .or_else(|| message[\"reasoning\"].as_str())\n .or_else(|| message[\"reasoning_content\"].as_str());\n\n text.map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"no content in VLM response: {}\", response_json))\n}\n\nconst OCRS_DETECTION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten\";\nconst OCRS_RECOGNITION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten\";\n\nfn ocrs_cache_dir() -> Result<PathBuf> {\n let base_cache = std::env::var(\"XDG_CACHE_HOME\")\n .ok()\n .filter(|v| !v.trim().is_empty())\n .map(PathBuf::from)\n .or_else(dirs::cache_dir)\n .unwrap_or_else(|| {\n std::env::var(\"HOME\")\n .map(PathBuf::from)\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\".cache\")\n });\n\n let cache_dir = base_cache.join(\"ocrs\");\n fs::create_dir_all(&cache_dir)\n .with_context(|| format!(\"creating OCRS cache dir {}\", cache_dir.display()))?;\n Ok(cache_dir)\n}\n\nfn human_bytes(bytes: u64) -> String {\n const KB: f64 = 1024.0;\n const MB: f64 = KB * 1024.0;\n const GB: f64 = MB * 1024.0;\n\n let b = bytes as f64;\n if b >= GB {\n format!(\"{:.2} GB\", b / GB)\n } else if b >= MB {\n format!(\"{:.2} MB\", b / MB)\n } else if b >= KB {\n format!(\"{:.1} KB\", b / KB)\n } else {\n format!(\"{} B\", bytes)\n }\n}\n\nfn download_to_cache(url: &str, filename: &str) -> Result<PathBuf> {\n let path = ocrs_cache_dir()?.join(filename);\n if path.exists() {\n eprintln!(\"OCRS model cached: {}\", path.display());\n return Ok(path);\n }\n\n info!(\"downloading OCRS model from {}\", url);\n let mut response = reqwest::blocking::get(url)\n .with_context(|| format!(\"downloading model from {}\", url))?\n .error_for_status()\n .with_context(|| format!(\"failed to download model from {}\", url))?;\n\n let total = response.content_length();\n eprintln!(\n \"Downloading OCRS model {}{}\",\n filename,\n total\n .map(|t| format!(\" ({})\", human_bytes(t)))\n .unwrap_or_default()\n );\n\n let mut file = fs::File::create(&path)\n .with_context(|| format!(\"creating model file {}\", path.display()))?;\n let mut downloaded: u64 = 0;\n let mut buffer = vec![0_u8; 64 * 1024];\n let mut last_print = Instant::now();\n let started = Instant::now();\n\n loop {\n let n = response\n .read(&mut buffer)\n .with_context(|| format!(\"reading model bytes from {}\", url))?;\n if n == 0 {\n break;\n }\n\n file.write_all(&buffer[..n])\n .with_context(|| format!(\"writing model to {}\", path.display()))?;\n downloaded += n as u64;\n\n if last_print.elapsed() >= Duration::from_millis(200) {\n match total {\n Some(t) if t > 0 => {\n let pct = (downloaded as f64 / t as f64) * 100.0;\n eprint!(\n \"\\r -> {} / {} ({:.1}%)\",\n human_bytes(downloaded),\n human_bytes(t),\n pct\n );\n }\n _ => {\n eprint!(\"\\r -> downloaded {}\", human_bytes(downloaded));\n }\n }\n let _ = io::stderr().flush();\n last_print = Instant::now();\n }\n }\n\n let elapsed = started.elapsed().as_secs_f32();\n eprintln!(\n \"\\r -> done: {} in {:.1}s \",\n human_bytes(downloaded),\n elapsed\n );\n\n Ok(path)\n}\n\nfn run_ocrs_on_image(input: &Path, engine: &OcrEngine, show_progress: bool) -> Result<String> {\n let started = Instant::now();\n\n if show_progress {\n eprintln!(\"OCRS: loading image {}\", input.display());\n }\n let image = image::open(input)\n .with_context(|| format!(\"reading image for OCRS: {}\", input.display()))?\n .into_rgb8();\n\n let image_source = ImageSource::from_bytes(image.as_raw(), image.dimensions())\n .context(\"creating OCRS image source\")?;\n\n if show_progress {\n eprintln!(\"OCRS: preparing input tensor\");\n }\n let ocr_input = engine.prepare_input(image_source).context(\"preparing OCRS input\")?;\n\n if show_progress {\n eprintln!(\"OCRS: detecting words\");\n }\n let word_rects = engine.detect_words(&ocr_input).context(\"OCRS word detection failed\")?;\n\n if show_progress {\n eprintln!(\"OCRS: grouping into lines\");\n }\n let line_rects = engine.find_text_lines(&ocr_input, &word_rects);\n\n if show_progress {\n eprintln!(\"OCRS: recognizing text\");\n }\n let line_texts = engine\n .recognize_text(&ocr_input, &line_rects)\n .context(\"OCRS text recognition failed\")?;\n\n let text = line_texts\n .iter()\n .flatten()\n .map(|line| line.to_string())\n .filter(|line| !line.trim().is_empty())\n .collect::<Vec<_>>()\n .join(\"\\n\");\n\n if show_progress {\n eprintln!(\n \"OCRS: done in {:.2}s ({} words, {} lines)\",\n started.elapsed().as_secs_f32(),\n word_rects.len(),\n line_rects.len()\n );\n }\n\n Ok(text)\n}\n\nfn run_ocrs_on_pdf(input: &Path, engine: &OcrEngine) -> Result<String> {\n let temp_dir = std::env::temp_dir().join(format!(\"ingestr-ocrs-pdf-{}\", std::process::id()));\n if temp_dir.exists() {\n let _ = fs::remove_dir_all(&temp_dir);\n }\n fs::create_dir_all(&temp_dir)\n .with_context(|| format!(\"creating temp dir {}\", temp_dir.display()))?;\n\n let result = (|| -> Result<String> {\n eprint!(\"OCRS: rendering PDF pages...\");\n let render_started = Instant::now();\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total = page_images.len();\n eprintln!(\" {} pages in {:.1}s\", total, render_started.elapsed().as_secs_f32());\n\n let mut sections = Vec::with_capacity(total);\n let pipeline_started = Instant::now();\n\n for (idx, page) in page_images.iter().enumerate() {\n let page_idx = idx + 1;\n let page_started = Instant::now();\n let page_text = run_ocrs_on_image(page, engine, false)\n .with_context(|| format!(\"OCRS failed on PDF page {}\", page_idx))?;\n sections.push(format!(\"## Page {}\\n\\n{}\", page_idx, page_text));\n\n let done = page_idx;\n let avg = pipeline_started.elapsed().as_secs_f32() / done as f32;\n let remaining_pages = total.saturating_sub(done);\n let eta = avg * remaining_pages as f32;\n eprintln!(\n \"OCRS PDF: [{}/{}] page {} done ({:.2}s page, {:.2}s avg, ~{:.0}s remaining)\",\n done,\n total,\n page_idx,\n page_started.elapsed().as_secs_f32(),\n avg,\n eta\n );\n }\n\n eprintln!(\n \"OCRS PDF: completed {} pages in {:.1}s\",\n total,\n pipeline_started.elapsed().as_secs_f32()\n );\n\n Ok(sections.join(\"\\n\\n\"))\n })();\n\n let _ = fs::remove_dir_all(&temp_dir);\n result\n}\n\nfn run_ocrs(input: &Path, languages: &[String]) -> Result<String> {\n if !languages.is_empty() && languages.iter().all(|l| l != \"eng\") {\n warn!(\n \"ocrs backend currently works best for Latin/English; requested languages: {:?}\",\n languages\n );\n }\n\n let init_started = Instant::now();\n eprintln!(\"OCRS: preparing models\");\n let detection_model_path = download_to_cache(OCRS_DETECTION_MODEL_URL, \"text-detection.rten\")?;\n let recognition_model_path =\n download_to_cache(OCRS_RECOGNITION_MODEL_URL, \"text-recognition.rten\")?;\n\n eprintln!(\"OCRS: loading detection model {}\",&detection_model_path.display());\n let detection_model =\n RtenModel::load_file(&detection_model_path).context(\"loading OCRS detection model\")?;\n\n eprintln!(\"OCRS: loading recognition model {}\",&recognition_model_path.display());\n let recognition_model =\n RtenModel::load_file(&recognition_model_path).context(\"loading OCRS recognition model\")?;\n\n let engine = OcrEngine::new(OcrEngineParams {\n detection_model: Some(detection_model),\n recognition_model: Some(recognition_model),\n ..Default::default()\n })\n .context(\"initializing OCRS engine\")?;\n eprintln!(\"OCRS: models ready in {:.2}s\", init_started.elapsed().as_secs_f32());\n\n if input\n .extension()\n .and_then(|e| e.to_str())\n .is_some_and(|ext| ext.eq_ignore_ascii_case(\"pdf\"))\n {\n run_ocrs_on_pdf(input, &engine)\n } else {\n run_ocrs_on_image(input, &engine, true)\n }\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n\n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Ocrs => run_ocrs(input, languages),\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n\n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // Special patterns for batch conversion:\n // \":pptx\" -> find all .pptx files in current directory recursively and convert in-place\n if input_str.starts_with(':') {\n let ext = &input_str[1..]; // remove the leading ':'\n if ext.is_empty() {\n bail!(\"special pattern \":EXT\" requires an extension, e.g., \":pptx\"\");\n }\n let cmd = ConvertCommand {\n extensions: Some(vec![ext.to_string()]),\n recursive: true,\n in_place: true,\n ..cmd\n };\n return handle_convert_directory(ctx, &cmd, &settings, &env::current_dir()?);\n }\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Determine if we should write alongside source files\n let in_place = cmd.in_place || (output_dir.is_none() && !ctx.common.dry_run);\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n if !ctx.common.quiet {\n println!(\"Found {} files to convert\", total);\n }\n\n if ctx.common.dry_run {\n for file in &files {\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out.display().to_string()\n } else if let Some(ref out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n // Single-threaded mode gets interactive progress\n let show_progress = parallel == 1 && !ctx.common.quiet && !ctx.common.no_progress;\n let progress_counter = AtomicUsize::new(0);\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().enumerate().map(|(idx, file)| {\n // Show progress before processing\n if show_progress {\n let num = idx + 1;\n let file_name = file.file_name()\n .and_then(|n| n.to_str())\n .unwrap_or(\"unknown\");\n eprint!(\"\\r\\x1b[K[{}/{}] Converting {}...\", num, total, file_name);\n let _ = io::stderr().flush();\n }\n\n let result = process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n in_place,\n &converted,\n &failed,\n &errors,\n );\n\n // Show result indicator\n if show_progress {\n match &result {\n Ok(_) => eprint!(\" ✓\"),\n Err(_) => eprint!(\" ✗\"),\n }\n }\n\n result\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n // Add newline after progress indicator if used\n if show_progress {\n eprintln!();\n }\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n in_place: bool,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n // Determine output path: in-place (source dir), output dir, or stdout\n let output_path = if in_place {\n let mut out = file.to_path_buf();\n out.set_extension(\"md\");\n out\n } else if let Some(out_dir) = output_dir {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if in_place or output_dir is specified\n if in_place || output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Ocrs.to_string(), \"ocrs\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Ocrs);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "43a47508456798b725c49d76b425ce890bc7a712": {"t": 1785757405.0185702, "n": 146743, "h": "c48ca308235ce370", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, Instant, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse ocrs::{ImageSource, OcrEngine, OcrEngineParams};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse rten::Model as RtenModel;\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin)\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Write output to a file or directory instead of stdout\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"ocrs\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n Tesseract,\n #[default]\n Ocrs,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Ocrs => write!(f, \"ocrs\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n // lopdf can emit repetitive \"corrupt deflate stream\" warnings for encrypted/corrupt PDFs.\n // We handle these cases explicitly and return user-friendly errors, so hide this crate-level noise.\n builder.filter_module(\"lopdf\", LevelFilter::Error);\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, ocrs, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Ocrs,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Ocrs,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Check for encrypted PDFs before passing to markitdown to avoid panics\n if is_pdf_extension(&extension) {\n match is_pdf_encrypted(input) {\n Ok(true) => {\n // Encrypted PDF - try OCR if available, otherwise return error\n if self.ocr_enabled {\n if let Ok(ocr_result) = self.process_with_ocr(input) {\n if !ocr_result.text_content.trim().is_empty() {\n return Ok(ocr_result);\n }\n }\n }\n bail!(\n \"PDF is encrypted/password-protected and cannot be converted. \\\n Consider using --vlm flag to process it via a vision model, \\\n or --ocr flag to extract text via OCR.\"\n );\n }\n Ok(false) => {} // Not encrypted, proceed with markitdown\n Err(e) => {\n warn!(\"Could not check PDF encryption status: {}, attempting conversion anyway\", e);\n }\n }\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string())\n .trim_end_matches('/')\n .to_string();\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\n/// Check if a PDF file is encrypted/password-protected.\n/// Returns true if encrypted, false if not, or an error if the file cannot be read.\nfn is_pdf_encrypted(path: &Path) -> Result<bool> {\n use lopdf::{Document, Object};\n\n let doc = Document::load(path)\n .with_context(|| format!(\"failed to load PDF: {}\", path.display()))?;\n\n // Check the Encrypt dictionary in trailer\n if let Ok(encrypt) = doc.trailer.get(b\"Encrypt\") {\n if *encrypt != Object::Null {\n return Ok(true);\n }\n }\n\n Ok(false)\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096,\n // Disable thinking/reasoning for models that support it (Qwen3.5, etc.)\n // This puts the full response in \"content\" instead of \"reasoning\"\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n let message = &response_json[\"choices\"][0][\"message\"];\n\n // Try content first, then reasoning (for thinking models like Qwen3.5)\n let text = message[\"content\"]\n .as_str()\n .filter(|s| !s.is_empty())\n .or_else(|| message[\"reasoning\"].as_str())\n .or_else(|| message[\"reasoning_content\"].as_str());\n\n text.map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"no content in VLM response: {}\", response_json))\n}\n\nconst OCRS_DETECTION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten\";\nconst OCRS_RECOGNITION_MODEL_URL: &str = \"https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten\";\n\nfn ocrs_cache_dir() -> Result<PathBuf> {\n let base_cache = std::env::var(\"XDG_CACHE_HOME\")\n .ok()\n .filter(|v| !v.trim().is_empty())\n .map(PathBuf::from)\n .or_else(dirs::cache_dir)\n .unwrap_or_else(|| {\n std::env::var(\"HOME\")\n .map(PathBuf::from)\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\".cache\")\n });\n\n let cache_dir = base_cache.join(\"ocrs\");\n fs::create_dir_all(&cache_dir)\n .with_context(|| format!(\"creating OCRS cache dir {}\", cache_dir.display()))?;\n Ok(cache_dir)\n}\n\nfn human_bytes(bytes: u64) -> String {\n const KB: f64 = 1024.0;\n const MB: f64 = KB * 1024.0;\n const GB: f64 = MB * 1024.0;\n\n let b = bytes as f64;\n if b >= GB {\n format!(\"{:.2} GB\", b / GB)\n } else if b >= MB {\n format!(\"{:.2} MB\", b / MB)\n } else if b >= KB {\n format!(\"{:.1} KB\", b / KB)\n } else {\n format!(\"{} B\", bytes)\n }\n}\n\nfn download_to_cache(url: &str, filename: &str) -> Result<PathBuf> {\n let path = ocrs_cache_dir()?.join(filename);\n if path.exists() {\n eprintln!(\"OCRS model cached: {}\", path.display());\n return Ok(path);\n }\n\n info!(\"downloading OCRS model from {}\", url);\n let mut response = reqwest::blocking::get(url)\n .with_context(|| format!(\"downloading model from {}\", url))?\n .error_for_status()\n .with_context(|| format!(\"failed to download model from {}\", url))?;\n\n let total = response.content_length();\n eprintln!(\n \"Downloading OCRS model {}{}\",\n filename,\n total\n .map(|t| format!(\" ({})\", human_bytes(t)))\n .unwrap_or_default()\n );\n\n let mut file = fs::File::create(&path)\n .with_context(|| format!(\"creating model file {}\", path.display()))?;\n let mut downloaded: u64 = 0;\n let mut buffer = vec![0_u8; 64 * 1024];\n let mut last_print = Instant::now();\n let started = Instant::now();\n\n loop {\n let n = response\n .read(&mut buffer)\n .with_context(|| format!(\"reading model bytes from {}\", url))?;\n if n == 0 {\n break;\n }\n\n file.write_all(&buffer[..n])\n .with_context(|| format!(\"writing model to {}\", path.display()))?;\n downloaded += n as u64;\n\n if last_print.elapsed() >= Duration::from_millis(200) {\n match total {\n Some(t) if t > 0 => {\n let pct = (downloaded as f64 / t as f64) * 100.0;\n eprint!(\n \"\\r -> {} / {} ({:.1}%)\",\n human_bytes(downloaded),\n human_bytes(t),\n pct\n );\n }\n _ => {\n eprint!(\"\\r -> downloaded {}\", human_bytes(downloaded));\n }\n }\n let _ = io::stderr().flush();\n last_print = Instant::now();\n }\n }\n\n let elapsed = started.elapsed().as_secs_f32();\n eprintln!(\n \"\\r -> done: {} in {:.1}s \",\n human_bytes(downloaded),\n elapsed\n );\n\n Ok(path)\n}\n\nfn run_ocrs_on_image(input: &Path, engine: &OcrEngine, show_progress: bool) -> Result<String> {\n let started = Instant::now();\n\n if show_progress {\n eprintln!(\"OCRS: loading image {}\", input.display());\n }\n let image = image::open(input)\n .with_context(|| format!(\"reading image for OCRS: {}\", input.display()))?\n .into_rgb8();\n\n let image_source = ImageSource::from_bytes(image.as_raw(), image.dimensions())\n .context(\"creating OCRS image source\")?;\n\n if show_progress {\n eprintln!(\"OCRS: preparing input tensor\");\n }\n let ocr_input = engine.prepare_input(image_source).context(\"preparing OCRS input\")?;\n\n if show_progress {\n eprintln!(\"OCRS: detecting words\");\n }\n let word_rects = engine.detect_words(&ocr_input).context(\"OCRS word detection failed\")?;\n\n if show_progress {\n eprintln!(\"OCRS: grouping into lines\");\n }\n let line_rects = engine.find_text_lines(&ocr_input, &word_rects);\n\n if show_progress {\n eprintln!(\"OCRS: recognizing text\");\n }\n let line_texts = engine\n .recognize_text(&ocr_input, &line_rects)\n .context(\"OCRS text recognition failed\")?;\n\n let text = line_texts\n .iter()\n .flatten()\n .map(|line| line.to_string())\n .filter(|line| !line.trim().is_empty())\n .collect::<Vec<_>>()\n .join(\"\\n\");\n\n if show_progress {\n eprintln!(\n \"OCRS: done in {:.2}s ({} words, {} lines)\",\n started.elapsed().as_secs_f32(),\n word_rects.len(),\n line_rects.len()\n );\n }\n\n Ok(text)\n}\n\nfn run_ocrs_on_pdf(input: &Path, engine: &OcrEngine) -> Result<String> {\n let temp_dir = std::env::temp_dir().join(format!(\"ingestr-ocrs-pdf-{}\", std::process::id()));\n if temp_dir.exists() {\n let _ = fs::remove_dir_all(&temp_dir);\n }\n fs::create_dir_all(&temp_dir)\n .with_context(|| format!(\"creating temp dir {}\", temp_dir.display()))?;\n\n let result = (|| -> Result<String> {\n eprint!(\"OCRS: rendering PDF pages...\");\n let render_started = Instant::now();\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total = page_images.len();\n eprintln!(\" {} pages in {:.1}s\", total, render_started.elapsed().as_secs_f32());\n\n let mut sections = Vec::with_capacity(total);\n let pipeline_started = Instant::now();\n\n for (idx, page) in page_images.iter().enumerate() {\n let page_idx = idx + 1;\n let page_started = Instant::now();\n let page_text = run_ocrs_on_image(page, engine, false)\n .with_context(|| format!(\"OCRS failed on PDF page {}\", page_idx))?;\n sections.push(format!(\"## Page {}\\n\\n{}\", page_idx, page_text));\n\n let done = page_idx;\n let avg = pipeline_started.elapsed().as_secs_f32() / done as f32;\n let remaining_pages = total.saturating_sub(done);\n let eta = avg * remaining_pages as f32;\n eprintln!(\n \"OCRS PDF: [{}/{}] page {} done ({:.2}s page, {:.2}s avg, ~{:.0}s remaining)\",\n done,\n total,\n page_idx,\n page_started.elapsed().as_secs_f32(),\n avg,\n eta\n );\n }\n\n eprintln!(\n \"OCRS PDF: completed {} pages in {:.1}s\",\n total,\n pipeline_started.elapsed().as_secs_f32()\n );\n\n Ok(sections.join(\"\\n\\n\"))\n })();\n\n let _ = fs::remove_dir_all(&temp_dir);\n result\n}\n\nfn run_ocrs(input: &Path, languages: &[String]) -> Result<String> {\n if !languages.is_empty() && languages.iter().all(|l| l != \"eng\") {\n warn!(\n \"ocrs backend currently works best for Latin/English; requested languages: {:?}\",\n languages\n );\n }\n\n let init_started = Instant::now();\n eprintln!(\"OCRS: preparing models\");\n let detection_model_path = download_to_cache(OCRS_DETECTION_MODEL_URL, \"text-detection.rten\")?;\n let recognition_model_path =\n download_to_cache(OCRS_RECOGNITION_MODEL_URL, \"text-recognition.rten\")?;\n\n eprintln!(\"OCRS: loading detection model {}\",&detection_model_path.display());\n let detection_model =\n RtenModel::load_file(&detection_model_path).context(\"loading OCRS detection model\")?;\n\n eprintln!(\"OCRS: loading recognition model {}\",&recognition_model_path.display());\n let recognition_model =\n RtenModel::load_file(&recognition_model_path).context(\"loading OCRS recognition model\")?;\n\n let engine = OcrEngine::new(OcrEngineParams {\n detection_model: Some(detection_model),\n recognition_model: Some(recognition_model),\n ..Default::default()\n })\n .context(\"initializing OCRS engine\")?;\n eprintln!(\"OCRS: models ready in {:.2}s\", init_started.elapsed().as_secs_f32());\n\n if input\n .extension()\n .and_then(|e| e.to_str())\n .is_some_and(|ext| ext.eq_ignore_ascii_case(\"pdf\"))\n {\n run_ocrs_on_pdf(input, &engine)\n } else {\n run_ocrs_on_image(input, &engine, true)\n }\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n\n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Ocrs => run_ocrs(input, languages),\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n\n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n info!(\"found {} files to convert\", total);\n\n if ctx.common.dry_run {\n for file in &files {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(ref out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if output_dir is specified\n if output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Ocrs.to_string(), \"ocrs\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Ocrs);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "c5901959e4f6da711879c51440a34eb19e465932": {"t": 1785757407.1459446, "n": 135316, "h": "02e9d49c4f9288cd", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin)\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Write output to a file or directory instead of stdout\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"tesseract\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n #[default]\n Tesseract,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Tesseract,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Tesseract,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string())\n .trim_end_matches('/')\n .to_string();\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096,\n // Disable thinking/reasoning for models that support it (Qwen3.5, etc.)\n // This puts the full response in \"content\" instead of \"reasoning\"\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n let message = &response_json[\"choices\"][0][\"message\"];\n\n // Try content first, then reasoning (for thinking models like Qwen3.5)\n let text = message[\"content\"]\n .as_str()\n .filter(|s| !s.is_empty())\n .or_else(|| message[\"reasoning\"].as_str())\n .or_else(|| message[\"reasoning_content\"].as_str());\n\n text.map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"no content in VLM response: {}\", response_json))\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n \n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n \n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n info!(\"found {} files to convert\", total);\n\n if ctx.common.dry_run {\n for file in &files {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(ref out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if output_dir is specified\n if output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Tesseract);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "c3f708fbd0c0ee3733f75062acd160abc48fda8a": {"t": 1785757408.945973, "n": 134768, "h": "c5d448d87c0d74a4", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse rayon::prelude::*;\nuse regex::Regex;\nuse serde::{Deserialize, Serialize};\nuse sha2::{Digest, Sha256};\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\n/// Known subcommand names (used for default-subcommand detection).\nconst KNOWN_SUBCOMMANDS: &[&str] = &[\n \"service\", \"search\", \"convert\", \"init\", \"config\", \"cache\", \"completions\", \"help\",\n];\n\nfn try_main() -> Result<()> {\n // Default subcommand: if the first positional arg is not a known subcommand,\n // treat it as `convert <INPUT>`. This lets users write `ingestr file.pdf`\n // instead of `ingestr convert file.pdf`.\n let args: Vec<String> = env::args().collect();\n let cli = if args.len() > 1 {\n let first_arg = &args[1];\n // Skip if it's a flag (starts with -)\n let is_flag = first_arg.starts_with('-');\n let is_subcommand = KNOWN_SUBCOMMANDS.contains(&first_arg.as_str());\n if !is_flag && !is_subcommand {\n // Insert \"convert\" as the subcommand\n let mut new_args = vec![args[0].clone(), \"convert\".to_string()];\n new_args.extend_from_slice(&args[1..]);\n Cli::parse_from(new_args)\n } else {\n Cli::parse()\n }\n } else {\n Cli::parse()\n };\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Cache { command } => handle_cache(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Manage the conversion cache\n Cache {\n #[command(subcommand)]\n command: CacheCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Subcommand)]\nenum CacheCommand {\n /// Clear the conversion cache\n Clear,\n /// Show cache statistics\n Stats,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File, directory, or URL to convert (use '-' or omit for stdin)\n #[arg(value_name = \"INPUT\")]\n input: Option<String>,\n /// Write output to a file or directory instead of stdout\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Vision model to use (e.g., \"glm-4.6v-flash\", \"qwen/qwen3-vl-8b\")\n #[arg(long, value_name = \"MODEL\")]\n vlm_model: Option<String>,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Number of parallel VLM requests (default: 1)\n #[arg(short = 'j', long = \"jobs\", value_name = \"N\", default_value = \"1\")]\n jobs: usize,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"tesseract\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n /// Include YAML frontmatter with metadata in output\n #[arg(long)]\n meta: bool,\n /// Skip post-processing cleanup (output raw conversion result)\n #[arg(long)]\n raw: bool,\n /// Maximum tokens to output (approximate, truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_tokens: Option<usize>,\n /// Maximum characters to output (truncates at section boundaries)\n #[arg(long, value_name = \"N\")]\n max_chars: Option<usize>,\n /// Character offset to start from (for pagination with --max-tokens/--max-chars)\n #[arg(long, value_name = \"N\", default_value_t = 0)]\n offset: usize,\n /// Show document structure (table of contents with token estimates)\n #[arg(long)]\n toc: bool,\n /// Extract only a specific section by number or heading (e.g., \"2.1\" or \"Summary\")\n #[arg(short = 's', long, value_name = \"SECTION\")]\n section: Option<String>,\n /// Convert only specific pages (PDF only, e.g., \"1-3,7\")\n #[arg(long, value_name = \"PAGES\")]\n pages: Option<String>,\n /// Read input from the system clipboard\n #[arg(long)]\n clipboard: bool,\n /// Bypass the conversion cache\n #[arg(long)]\n no_cache: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n #[default]\n Tesseract,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration.\n/// When `llm_url` or `model` are left empty/default, the main `[llm]` config\n/// is used as fallback so users only need to configure their LLM once.\n/// Works with any OpenAI-compatible vision API (Ollama, LM Studio, vLLM, etc.).\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// LLM server URL (falls back to [llm].base_url if empty)\n llm_url: Option<String>,\n /// Model name (falls back to [llm].model if empty)\n model: Option<String>,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n llm_url: None,\n model: None,\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Tesseract,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n /// If true, content was already streamed to a file during VLM processing\n already_written: bool,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n already_written: false,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n// ============================================================\n// Post-processing: clean noisy conversion output\n// ============================================================\n\n/// Estimate token count (rough: ~4 chars per token for English text)\nfn estimate_tokens(text: &str) -> usize {\n // A simple heuristic: split on whitespace, count words,\n // then apply ~1.3 tokens per word (common for English).\n // For non-English or code-heavy content, chars/4 is more stable.\n (text.len() + 3) / 4\n}\n\n/// Clean converted markdown by removing common PDF/document noise.\nfn clean_markdown(text: &str) -> String {\n let mut lines: Vec<&str> = text.lines().collect();\n\n // 1. Detect and remove repeated headers/footers.\n // If the same line appears on 3+ \"pages\" (roughly every 40-80 lines), it is likely a header/footer.\n if lines.len() > 80 {\n let mut freq: HashMap<String, usize> = HashMap::new();\n for line in &lines {\n let trimmed = line.trim();\n if !trimmed.is_empty() && trimmed.len() < 120 {\n *freq.entry(trimmed.to_lowercase()).or_insert(0) += 1;\n }\n }\n let threshold = (lines.len() / 60).max(3);\n let repeated: std::collections::HashSet<String> = freq\n .into_iter()\n .filter(|(_, count)| *count >= threshold)\n .map(|(line, _)| line)\n .collect();\n\n if !repeated.is_empty() {\n lines.retain(|line| {\n let trimmed = line.trim().to_lowercase();\n !repeated.contains(&trimmed)\n });\n }\n }\n\n // 2. Remove standalone page numbers (lines that are just a number, optionally with \"Page\" prefix)\n let page_num_re = Regex::new(r\"(?i)^\\s*(page\\s+)?\\d+(\\s+of\\s+\\d+)?\\s*$\").unwrap();\n lines.retain(|line| !page_num_re.is_match(line));\n\n // 3. Fix broken line wraps from PDF column layouts:\n // If a line ends without punctuation or a heading marker and the next starts lowercase, join them.\n let mut result = String::with_capacity(text.len());\n let mut i = 0;\n while i < lines.len() {\n let current = lines[i].trim_end();\n if i + 1 < lines.len() {\n let next = lines[i + 1].trim_start();\n let current_trimmed = current.trim();\n\n // Join if: current doesn't end with sentence-ender or heading,\n // current is not empty, next starts with lowercase\n let should_join = !current_trimmed.is_empty()\n && !next.is_empty()\n && !current_trimmed.ends_with('.')\n && !current_trimmed.ends_with(':')\n && !current_trimmed.ends_with('!')\n && !current_trimmed.ends_with('?')\n && !current_trimmed.ends_with('|')\n && !current_trimmed.starts_with('#')\n && !current_trimmed.starts_with('-')\n && !current_trimmed.starts_with('*')\n && !current_trimmed.starts_with('|')\n && !next.starts_with('#')\n && !next.starts_with('-')\n && !next.starts_with('*')\n && !next.starts_with('|')\n && !next.starts_with('>')\n && next.starts_with(|c: char| c.is_lowercase());\n\n if should_join {\n result.push_str(current_trimmed);\n result.push(' ');\n i += 1;\n continue;\n }\n }\n result.push_str(current);\n result.push('\\n');\n i += 1;\n }\n\n // 4. Collapse 3+ consecutive blank lines into 2\n let multi_blank = Regex::new(r\"\\n{4,}\").unwrap();\n let result = multi_blank.replace_all(&result, \"\\n\\n\\n\").to_string();\n\n // 5. Trim leading/trailing whitespace\n result.trim().to_string()\n}\n\n// ============================================================\n// Document structure: TOC extraction and section retrieval\n// ============================================================\n\n#[derive(Debug, Clone, Serialize)]\nstruct TocEntry {\n /// Heading level (1-6)\n level: usize,\n /// Section number (e.g., \"2.1.3\")\n number: String,\n /// Heading text\n title: String,\n /// Character offset where this section starts\n char_start: usize,\n /// Character offset where this section ends\n char_end: usize,\n /// Estimated token count for this section\n tokens: usize,\n /// Whether this section contains a markdown table\n has_table: bool,\n /// Whether this section contains a code block\n has_code: bool,\n}\n\n/// Extract table of contents from markdown content.\nfn extract_toc(content: &str) -> Vec<TocEntry> {\n let heading_re = Regex::new(r\"^(#{1,6})\\s+(.+)$\").unwrap();\n let mut entries: Vec<TocEntry> = Vec::new();\n let mut counters: Vec<usize> = vec![0; 7]; // index 1-6 for heading levels\n\n // Find all heading positions\n let mut heading_positions: Vec<(usize, usize, String)> = Vec::new();\n for (offset, line) in content.lines().scan(0usize, |pos, line| {\n let start = *pos;\n *pos += line.len() + 1; // +1 for newline\n Some((start, line))\n }) {\n if let Some(caps) = heading_re.captures(line) {\n let level = caps[1].len();\n let title = caps[2].trim().to_string();\n heading_positions.push((offset, level, title));\n }\n }\n\n for (idx, (char_start, level, title)) in heading_positions.iter().enumerate() {\n let level = *level;\n let char_start = *char_start;\n\n // Calculate section end: either start of next heading at same or higher level, or end of doc\n let char_end = heading_positions\n .iter()\n .skip(idx + 1)\n .find(|(_, l, _)| *l <= level)\n .map(|(pos, _, _)| *pos)\n .unwrap_or(content.len());\n\n let section_text = &content[char_start..char_end];\n\n // Update counters for numbering\n counters[level] += 1;\n // Reset all deeper counters\n for c in counters.iter_mut().skip(level + 1) {\n *c = 0;\n }\n\n // Build section number\n let number: String = counters[1..=level]\n .iter()\n .map(|n| n.to_string())\n .collect::<Vec<_>>()\n .join(\".\");\n\n let tokens = estimate_tokens(section_text);\n let has_table = section_text.contains(\"\\n|\") && section_text.contains(\"---|\");\n let has_code = section_text.contains(\"```\");\n\n entries.push(TocEntry {\n level,\n number,\n title: title.clone(),\n char_start,\n char_end,\n tokens,\n has_table,\n has_code,\n });\n }\n\n entries\n}\n\n/// Format TOC for display\nfn format_toc(toc: &[TocEntry], total_tokens: usize) -> String {\n let mut out = String::new();\n for entry in toc {\n let indent = \" \".repeat(entry.level.saturating_sub(1));\n let mut markers = Vec::new();\n if entry.has_table {\n markers.push(\"table\");\n }\n if entry.has_code {\n markers.push(\"code\");\n }\n let marker_str = if markers.is_empty() {\n String::new()\n } else {\n format!(\", has {}\", markers.join(\"+\"))\n };\n out.push_str(&format!(\n \"{}{}. {} (tokens: ~{}{})\\n\",\n indent, entry.number, entry.title, entry.tokens, marker_str\n ));\n }\n out.push_str(&format!(\"\\nTotal: ~{} tokens across {} sections\\n\", total_tokens, toc.len()));\n out\n}\n\n/// Extract a specific section by number (e.g., \"2.1\") or by heading text (fuzzy match).\nfn extract_section(content: &str, selector: &str) -> Option<String> {\n let toc = extract_toc(content);\n if toc.is_empty() {\n return None;\n }\n\n // Try exact number match first\n if let Some(entry) = toc.iter().find(|e| e.number == selector) {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n // Try heading text match (case-insensitive substring)\n let selector_lower = selector.to_lowercase();\n if let Some(entry) = toc\n .iter()\n .find(|e| e.title.to_lowercase().contains(&selector_lower))\n {\n return Some(content[entry.char_start..entry.char_end].to_string());\n }\n\n None\n}\n\n/// Truncate content at a section boundary, respecting a character budget.\nfn truncate_at_boundary(content: &str, max_chars: usize, offset: usize) -> (String, Option<String>) {\n if offset >= content.len() {\n return (String::new(), None);\n }\n\n let sliced = &content[offset..];\n if sliced.len() <= max_chars {\n return (sliced.to_string(), None);\n }\n\n // Find the last heading boundary before max_chars\n let heading_re = Regex::new(r\"\\n#{1,6}\\s\").unwrap();\n let mut last_break = max_chars;\n\n for m in heading_re.find_iter(&sliced[..max_chars]) {\n last_break = m.start();\n }\n\n // If no heading found, try paragraph break\n if last_break == max_chars {\n if let Some(pos) = sliced[..max_chars].rfind(\"\\n\\n\") {\n last_break = pos;\n }\n }\n\n let truncated = sliced[..last_break].trim_end().to_string();\n let remaining_chars = sliced.len() - last_break;\n let remaining_tokens = estimate_tokens(&sliced[last_break..]);\n\n let toc = extract_toc(content);\n let total_sections = toc.len();\n let sections_shown = extract_toc(&content[..offset + last_break]).len();\n\n let notice = format!(\n \"\\n\\n[truncated at char {}, ~{} tokens and ~{} chars remaining, sections {}/{}]\",\n offset + last_break,\n remaining_tokens,\n remaining_chars,\n sections_shown,\n total_sections\n );\n\n (truncated, Some(notice))\n}\n\n/// Filter content by page numbers. Looks for page break markers or splits by rough page boundaries.\nfn filter_pages(content: &str, pages: &[usize]) -> String {\n // Many PDF converters insert form-feed (\\x0c) or \"---\" page breaks.\n // Also look for patterns like \"Page N\" or just form-feeds.\n let page_break = Regex::new(r\"(?:\\x0c|\\n-{3,}\\n|\\n\\* \\* \\*\\n)\").unwrap();\n\n let page_texts: Vec<&str> = page_break.split(content).collect();\n\n if page_texts.len() <= 1 {\n // No page breaks found - if the user asked for page 1, return everything\n if pages.contains(&1) {\n return content.to_string();\n }\n // Otherwise, we can't split by pages without markers\n return format!(\n \"{}\\n\\n[note: no page break markers found in document, showing all content]\",\n content\n );\n }\n\n let mut result = String::new();\n for &page_num in pages {\n if page_num > 0 && page_num <= page_texts.len() {\n if !result.is_empty() {\n result.push_str(\"\\n\\n\");\n }\n result.push_str(page_texts[page_num - 1].trim());\n }\n }\n\n if result.is_empty() {\n format!(\n \"[no content found for pages {:?}, document has {} pages]\",\n pages,\n page_texts.len()\n )\n } else {\n result\n }\n}\n\n/// Parse a page range string like \"1-3,7,10-12\" into a sorted list of page numbers.\nfn parse_page_range(spec: &str) -> Result<Vec<usize>> {\n let mut pages = Vec::new();\n for part in spec.split(',') {\n let part = part.trim();\n if part.contains('-') {\n let bounds: Vec<&str> = part.split('-').collect();\n if bounds.len() != 2 {\n bail!(\"invalid page range: {}\", part);\n }\n let start: usize = bounds[0].trim().parse().context(\"invalid page number\")?;\n let end: usize = bounds[1].trim().parse().context(\"invalid page number\")?;\n if start == 0 || end == 0 {\n bail!(\"page numbers start at 1\");\n }\n if start > end {\n bail!(\"invalid range: {} > {}\", start, end);\n }\n for p in start..=end {\n pages.push(p);\n }\n } else {\n let p: usize = part.parse().context(\"invalid page number\")?;\n if p == 0 {\n bail!(\"page numbers start at 1\");\n }\n pages.push(p);\n }\n }\n pages.sort();\n pages.dedup();\n Ok(pages)\n}\n\n// ============================================================\n// Conversion cache\n// ============================================================\n\nfn cache_dir() -> Result<PathBuf> {\n let dir = if let Some(cache) = env::var_os(\"XDG_CACHE_HOME\").filter(|v| !v.is_empty()) {\n PathBuf::from(cache).join(\"ingestr\")\n } else if let Some(home) = dirs::home_dir() {\n home.join(\".cache\").join(\"ingestr\")\n } else {\n bail!(\"unable to determine cache directory\");\n };\n Ok(dir)\n}\n\n/// Compute a cache key from file hash + conversion flags.\nfn cache_key(path: &Path, meta: bool, raw: bool, vlm: bool, ocr: bool, section: &Option<String>, pages: &Option<String>) -> Result<String> {\n let data = fs::read(path).context(\"reading file for cache key\")?;\n let mut hasher = Sha256::new();\n hasher.update(&data);\n // Include flags in hash so different flag combos get different cache entries\n hasher.update(if meta { b\"meta=1\" } else { b\"meta=0\" });\n hasher.update(if raw { b\"raw=1\" } else { b\"raw=0\" });\n hasher.update(if vlm { b\"vlm=1\" } else { b\"vlm=0\" });\n hasher.update(if ocr { b\"ocr=1\" } else { b\"ocr=0\" });\n if let Some(s) = section {\n hasher.update(format!(\"section={s}\").as_bytes());\n }\n if let Some(p) = pages {\n hasher.update(format!(\"pages={p}\").as_bytes());\n }\n let hash = hasher.finalize();\n Ok(hex::encode(hash))\n}\n\n/// Try to read a cached conversion result.\nfn cache_get(key: &str) -> Result<Option<String>> {\n let path = cache_dir()?.join(&key[..2]).join(key);\n if path.exists() {\n Ok(Some(fs::read_to_string(path)?))\n } else {\n Ok(None)\n }\n}\n\n/// Store a conversion result in the cache.\nfn cache_put(key: &str, content: &str) -> Result<()> {\n let dir = cache_dir()?.join(&key[..2]);\n fs::create_dir_all(&dir)?;\n fs::write(dir.join(key), content)?;\n Ok(())\n}\n\nfn handle_cache(ctx: &RuntimeContext, command: CacheCommand) -> Result<()> {\n match command {\n CacheCommand::Clear => {\n let dir = cache_dir()?;\n if dir.exists() {\n let count = WalkDir::new(&dir)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .count();\n if ctx.common.dry_run {\n info!(\"dry-run: would remove {} cached files from {}\", count, dir.display());\n } else {\n fs::remove_dir_all(&dir)?;\n fs::create_dir_all(&dir)?;\n println!(\"Cleared {} cached conversions\", count);\n }\n } else {\n println!(\"Cache is empty\");\n }\n Ok(())\n }\n CacheCommand::Stats => {\n let dir = cache_dir()?;\n if !dir.exists() {\n if ctx.common.json {\n println!(r#\"{{\"files\": 0, \"size_bytes\": 0, \"path\": \"{}\"}}\"#, dir.display());\n } else {\n println!(\"Cache is empty ({})\", dir.display());\n }\n return Ok(());\n }\n let mut files = 0usize;\n let mut total_size = 0u64;\n for entry in WalkDir::new(&dir).into_iter().filter_map(Result::ok) {\n if entry.file_type().is_file() {\n files += 1;\n total_size += entry.metadata().map(|m| m.len()).unwrap_or(0);\n }\n }\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&serde_json::json!({\n \"files\": files,\n \"size_bytes\": total_size,\n \"size_mb\": format!(\"{:.1}\", total_size as f64 / 1_048_576.0),\n \"path\": dir.display().to_string(),\n }))?\n );\n } else {\n println!(\n \"Cache: {} files, {:.1} MB ({})\",\n files,\n total_size as f64 / 1_048_576.0,\n dir.display()\n );\n }\n Ok(())\n }\n }\n}\n\n// ============================================================\n// URL fetching\n// ============================================================\n\n/// Check if a string looks like a URL\nfn is_url(input: &str) -> bool {\n input.starts_with(\"http://\") || input.starts_with(\"https://\")\n}\n\n/// Fetch a URL and save to a temp file for conversion.\n/// Returns the temp file path and a cleanup guard.\nfn fetch_url(url: &str) -> Result<PathBuf> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(30))\n .redirect(reqwest::redirect::Policy::limited(10))\n .user_agent(\"ingestr/0.1\")\n .build()\n .context(\"building HTTP client\")?;\n\n let response = client\n .get(url)\n .send()\n .with_context(|| format!(\"fetching URL: {url}\"))?;\n\n if !response.status().is_success() {\n bail!(\"HTTP {} fetching {}\", response.status(), url);\n }\n\n // Determine extension from Content-Type or URL\n let content_type = response\n .headers()\n .get(\"content-type\")\n .and_then(|v| v.to_str().ok())\n .unwrap_or(\"\")\n .to_lowercase();\n\n let ext = if content_type.contains(\"pdf\") {\n \"pdf\"\n } else if content_type.contains(\"html\") {\n \"html\"\n } else if content_type.contains(\"json\") {\n \"json\"\n } else if content_type.contains(\"xml\") {\n \"xml\"\n } else if content_type.contains(\"wordprocessingml\") || content_type.contains(\"docx\") {\n \"docx\"\n } else if content_type.contains(\"spreadsheetml\") || content_type.contains(\"xlsx\") {\n \"xlsx\"\n } else if content_type.contains(\"presentationml\") || content_type.contains(\"pptx\") {\n \"pptx\"\n } else {\n // Try to get extension from URL path\n url.rsplit('/')\n .next()\n .and_then(|segment| segment.rsplit('.').next())\n .filter(|ext| ext.len() <= 5 && ext.chars().all(|c| c.is_alphanumeric()))\n .unwrap_or(\"html\")\n };\n\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-url.{ext}\"));\n let bytes = response.bytes().context(\"reading response body\")?;\n fs::write(&temp_path, &bytes).context(\"writing fetched content to temp file\")?;\n\n Ok(temp_path)\n}\n\n// ============================================================\n// Clipboard support\n// ============================================================\n\n/// Read content from the system clipboard and save to a temp file.\nfn read_clipboard() -> Result<PathBuf> {\n let mut clipboard = arboard::Clipboard::new().context(\"accessing clipboard\")?;\n\n // Try to get text (which might be HTML or plain text)\n match clipboard.get_text() {\n Ok(text) if !text.trim().is_empty() => {\n let ext = detect_format_from_content(text.as_bytes());\n let temp_path = std::env::temp_dir().join(format!(\"ingestr-clipboard.{ext}\"));\n fs::write(&temp_path, &text).context(\"writing clipboard to temp file\")?;\n Ok(temp_path)\n }\n _ => {\n // Try image\n match clipboard.get_image() {\n Ok(img) => {\n let temp_path = std::env::temp_dir().join(\"ingestr-clipboard.png\");\n // arboard gives us raw RGBA data; write as PNG\n let mut png_data = Vec::new();\n let mut encoder = io::Cursor::new(&mut png_data);\n // Simple BMP-like approach: just write the raw bytes and let markitdown handle it\n // Actually, we need a proper image format. Let's write raw RGBA and use a simple approach.\n // For now, write a simple PPM which is easy to generate\n write!(\n encoder,\n \"P6\\n{} {}\\n255\\n\",\n img.width, img.height\n )?;\n for pixel in img.bytes.chunks(4) {\n encoder.write_all(&pixel[..3])?; // RGB, skip A\n }\n fs::write(&temp_path, &png_data).context(\"writing clipboard image\")?;\n // Rename to .ppm since that's what we wrote\n let ppm_path = std::env::temp_dir().join(\"ingestr-clipboard.ppm\");\n fs::rename(&temp_path, &ppm_path).ok();\n Ok(ppm_path)\n }\n Err(_) => bail!(\"clipboard is empty or contains unsupported content\"),\n }\n }\n }\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_model: Option<String>,\n vlm_prompt: Option<String>,\n jobs: usize,\n /// Output file for streaming VLM results (pages written as they complete)\n vlm_output: Option<PathBuf>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_model: None,\n vlm_prompt: None,\n jobs: 1,\n vlm_output: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Tesseract,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, model: Option<String>, prompt: Option<String>, jobs: usize, output: Option<PathBuf>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_model = model;\n self.vlm_prompt = prompt;\n self.jobs = jobs.max(1);\n self.vlm_output = output;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if VLM is enabled for images, PDFs, or presentations\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n if self.vlm_enabled && is_pdf_extension(&extension) {\n return self.process_pdf_with_vlm(input);\n }\n if self.vlm_enabled && is_presentation_extension(&extension) {\n return self.process_pptx_with_vlm(input);\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n already_written: false,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Resolve effective VLM connection settings.\n /// Priority: CLI --vlm-model > [processors.vlm].model > [llm].model\n /// Priority: [processors.vlm].llm_url > [llm].base_url > localhost:11434\n fn vlm_connection(&self) -> (String, String) {\n let vlm = &self.settings.processors.vlm;\n let url = vlm.llm_url.clone()\n .filter(|s| !s.is_empty())\n .or_else(|| self.settings.llm_base_url.clone())\n .unwrap_or_else(|| \"http://localhost:11434\".to_string());\n let model = self.vlm_model.clone()\n .or_else(|| vlm.model.clone())\n .filter(|s| !s.is_empty())\n .unwrap_or_else(|| self.settings.llm_model.clone());\n info!(\"VLM connection: url={}, model={}\", url, model);\n (url, model)\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call VLM API\n let (url, model) = self.vlm_connection();\n let description = call_vlm_api(&url, &model, &base64_image, mime_type, prompt)?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n already_written: false,\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n already_written: false,\n })\n }\n\n /// Convert a PDF page-by-page using VLM: render each page to an image with\n /// pdftoppm, send each image through the vision model, and concatenate the\n /// descriptions into a single markdown document.\n fn process_pdf_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this page in detail, including all visible text, tables, figures, and layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pdf-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Render PDF pages to PNG images using pdftoppm\n eprint!(\"Rendering PDF pages...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(input)\n .arg(temp_dir.join(\"page\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect page images sorted by name\n let mut page_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| p.extension().and_then(|e| e.to_str()) == Some(\"png\"))\n .collect();\n page_images.sort();\n\n if page_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no page images\");\n }\n\n let total_pages = page_images.len();\n let (url, model) = self.vlm_connection();\n eprintln!(\" {} pages (model: {})\", total_pages, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"document\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file: explicit -o, or default to <stem>.md in cwd\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"document\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n // Process pages with thread pool, streaming to file\n let jobs = self.jobs.min(total_pages);\n let sections = self.process_vlm_pages_parallel(\n &page_images, &url, &model, prompt, total_pages, jobs, \"Page\", &output_path,\n );\n\n // Clean up temp images\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version (replaces the streamed file)\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n\n /// Process page images through VLM in parallel, streaming results to a file\n /// as they complete (in order). Shows progress on stderr.\n fn process_vlm_pages_parallel(\n &self,\n page_images: &[PathBuf],\n url: &str,\n model: &str,\n prompt: &str,\n total: usize,\n jobs: usize,\n label: &str,\n output_path: &Path,\n ) -> Vec<String> {\n use std::sync::mpsc;\n use std::thread;\n\n let (tx, rx) = mpsc::channel::<(usize, String)>();\n\n // Read all images upfront so threads don't need filesystem access\n let images: Vec<(usize, Vec<u8>)> = page_images.iter().enumerate()\n .filter_map(|(i, path)| {\n fs::read(path).ok().map(|data| (i, data))\n })\n .collect();\n\n let url = url.to_string();\n let model = model.to_string();\n let prompt = prompt.to_string();\n\n // Spawn worker threads\n let chunk_size = (images.len() + jobs - 1) / jobs;\n let mut handles = Vec::new();\n\n for chunk in images.chunks(chunk_size) {\n let chunk: Vec<(usize, Vec<u8>)> = chunk.to_vec();\n let tx = tx.clone();\n let url = url.clone();\n let model = model.clone();\n let prompt = prompt.clone();\n\n let handle = thread::spawn(move || {\n for (page_idx, image_data) in chunk {\n let base64_image = base64_encode(&image_data);\n let result = match call_vlm_api(&url, &model, &base64_image, \"image/png\", &prompt) {\n Ok(desc) => desc,\n Err(e) => format!(\"[VLM processing failed: {}]\", e),\n };\n let _ = tx.send((page_idx, result));\n }\n });\n handles.push(handle);\n }\n drop(tx); // Close sender so rx iterator terminates\n\n // Collect results, append to file in order as pages become ready\n let mut results: Vec<Option<String>> = vec![None; total];\n let mut next_to_write = 0;\n let mut completed = 0;\n let start = std::time::Instant::now();\n let label = label.to_string();\n\n // Open file in append mode for streaming\n let mut file = fs::OpenOptions::new()\n .append(true)\n .open(output_path)\n .unwrap_or_else(|_| fs::File::create(output_path).expect(\"create output file\"));\n\n for (page_idx, description) in rx {\n completed += 1;\n let page_num = page_idx + 1;\n let elapsed = start.elapsed().as_secs_f32();\n let avg = elapsed / completed as f32;\n let remaining = avg * (total - completed) as f32;\n eprint!(\n \"\\r\\x1b[K[{}/{}] {} {} done ({:.1}s avg, ~{:.0}s remaining)\",\n completed, total, label, page_num, avg, remaining\n );\n\n let section = format!(\"## {} {}\\n\\n{}\", label, page_num, description);\n results[page_idx] = Some(section);\n\n // Append all consecutive ready pages to file\n while next_to_write < total {\n if let Some(ref s) = results[next_to_write] {\n let _ = writeln!(file, \"\\n{}\", s);\n let _ = file.flush();\n next_to_write += 1;\n } else {\n break;\n }\n }\n }\n\n // Wait for all threads\n for h in handles {\n let _ = h.join();\n }\n\n let elapsed = start.elapsed().as_secs_f32();\n eprintln!(\n \"\\r\\x1b[K[{}/{}] All {} pages done in {:.1}s -> {}\",\n total, total, label.to_lowercase(), elapsed, output_path.display()\n );\n\n results.into_iter().flatten().collect()\n }\n\n /// Convert a PPTX slide-by-slide using VLM: first convert to PDF with\n /// LibreOffice, then render each page to an image and send through VLM.\n fn process_pptx_with_vlm(&self, input: &Path) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"screenshot\"))\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this presentation slide in detail, including all text, diagrams, charts, images, bullet points, and visual layout.\");\n\n let temp_dir = std::env::temp_dir().join(\"ingestr-pptx-vlm\");\n fs::create_dir_all(&temp_dir)?;\n\n // Step 1: Convert to PDF using LibreOffice\n eprint!(\"Converting to PDF...\");\n let status = ProcCommand::new(\"soffice\")\n .args([\"--headless\", \"--convert-to\", \"pdf\", \"--outdir\"])\n .arg(&temp_dir)\n .arg(input)\n .status()\n .context(\"running LibreOffice (is soffice installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"LibreOffice conversion failed with status {}\", status);\n }\n\n // Find the generated PDF\n let pdf_path = {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .ok_or_else(|| anyhow!(\"invalid input filename\"))?;\n let pdf = temp_dir.join(format!(\"{}.pdf\", stem));\n if !pdf.exists() {\n // Try to find any PDF in the temp dir\n fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .find(|p| p.extension().and_then(|e| e.to_str()) == Some(\"pdf\"))\n .ok_or_else(|| anyhow!(\"LibreOffice produced no PDF output\"))?\n } else {\n pdf\n }\n };\n\n // Step 2: Render slides to images\n eprint!(\" Rendering slides...\");\n let status = ProcCommand::new(\"pdftoppm\")\n .args([\"-png\", \"-r\", \"200\"])\n .arg(&pdf_path)\n .arg(temp_dir.join(\"slide\"))\n .status()\n .context(\"running pdftoppm (is poppler installed?)\")?;\n\n if !status.success() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm failed with status {}\", status);\n }\n\n // Collect slide images sorted by name\n let mut slide_images: Vec<PathBuf> = fs::read_dir(&temp_dir)?\n .filter_map(Result::ok)\n .map(|e| e.path())\n .filter(|p| {\n p.file_name()\n .and_then(|n| n.to_str())\n .map(|n| n.starts_with(\"slide\") && n.ends_with(\".png\"))\n .unwrap_or(false)\n })\n .collect();\n slide_images.sort();\n\n if slide_images.is_empty() {\n let _ = fs::remove_dir_all(&temp_dir);\n bail!(\"pdftoppm produced no slide images\");\n }\n\n let (url, model) = self.vlm_connection();\n let total_slides = slide_images.len();\n eprintln!(\" {} slides (model: {})\", total_slides, model);\n\n let filename = input.file_name().and_then(|s| s.to_str()).unwrap_or(\"presentation\");\n let header = format!(\"# {}\\n\", filename);\n\n // Resolve output file\n let output_path = self.vlm_output.clone().unwrap_or_else(|| {\n let stem = input.file_stem()\n .and_then(|s| s.to_str())\n .unwrap_or(\"presentation\");\n PathBuf::from(format!(\"{}.md\", stem))\n });\n\n // Write header immediately\n fs::write(&output_path, &header)\n .with_context(|| format!(\"writing to {}\", output_path.display()))?;\n eprintln!(\"Streaming to {}\", output_path.display());\n\n let jobs = self.jobs.min(total_slides);\n let sections = self.process_vlm_pages_parallel(\n &slide_images, &url, &model, prompt, total_slides, jobs, \"Slide\", &output_path,\n );\n\n // Clean up temp files\n let _ = fs::remove_dir_all(&temp_dir);\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n let text_content = format!(\n \"{}\\n{}\\n\",\n header,\n sections.join(\"\\n\\n\")\n );\n\n // Write final clean version\n fs::write(&output_path, &text_content)\n .with_context(|| format!(\"writing final output to {}\", output_path.display()))?;\n\n Ok(ConvertedDocument { title, text_content, already_written: true })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn is_presentation_extension(ext: &str) -> bool {\n matches!(ext, \"pptx\" | \"ppt\" | \"odp\" | \"key\")\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call an OpenAI-compatible LLM API for vision processing.\n/// Works with Ollama, LM Studio, vLLM, or any server implementing\n/// the OpenAI /v1/chat/completions endpoint with vision support.\nfn call_vlm_api(\n llm_url: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::builder()\n .timeout(Duration::from_secs(120))\n .build()\n .context(\"building VLM HTTP client\")?;\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096\n });\n\n info!(\"VLM request to {}/v1/chat/completions model={}\", llm_url, model);\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", llm_url))\n .header(\"Content-Type\", \"application/json\")\n .json(&request_body)\n .send()\n .context(\"sending VLM request\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"VLM request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing VLM response\")?;\n\n response_json[\"choices\"][0][\"message\"][\"content\"]\n .as_str()\n .map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"invalid response format from VLM API\"))\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n \n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n \n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Clipboard input\n if cmd.clipboard {\n let temp_path = read_clipboard()?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, \"clipboard\")?;\n let _ = fs::remove_file(&temp_path);\n return result;\n }\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none()\n || cmd.input.as_ref().map(|s| s == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n let input_str = cmd.input.as_ref().unwrap().clone();\n\n // URL input\n if is_url(&input_str) {\n let temp_path = fetch_url(&input_str)?;\n let result = convert_single_input(ctx, &cmd, &settings, &temp_path, &input_str);\n let _ = fs::remove_file(&temp_path);\n return result?;\n }\n\n let input = expand_path(PathBuf::from(&input_str))?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let source_label = input.display().to_string();\n convert_single_input(ctx, &cmd, &settings, &input, &source_label)?\n}\n\n/// Core single-file conversion with all the new features:\n/// caching, cleaning, TOC, section extraction, token budget, frontmatter control.\nfn convert_single_input(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input: &Path,\n source_label: &str,\n) -> Result<Result<()>> {\n // Check cache first (unless --no-cache or --toc which is always computed fresh)\n if !cmd.no_cache && !cmd.toc && input.exists() {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n if let Ok(Some(cached)) = cache_get(&key) {\n debug!(\"cache hit for {}\", source_label);\n return Ok(output_final(ctx, cmd, cached, source_label));\n }\n }\n }\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(input)?;\n\n // Apply page filtering if --pages was specified\n let text_content = if let Some(ref page_spec) = cmd.pages {\n let pages = parse_page_range(page_spec)?;\n filter_pages(&converted.text_content, &pages)\n } else {\n converted.text_content.clone()\n };\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n text_content\n } else {\n clean_markdown(&text_content)\n };\n\n // TOC mode: just show structure and exit\n if cmd.toc {\n let toc = extract_toc(&content);\n let total_tokens = estimate_tokens(&content);\n if ctx.common.json {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let output = serde_json::json!({\n \"source\": source_label,\n \"total_tokens\": total_tokens,\n \"sections\": toc,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"{}\", format_toc(&toc, total_tokens));\n }\n return Ok(Ok(()));\n }\n\n // Section extraction\n let content = if let Some(ref selector) = cmd.section {\n extract_section(&content, selector)\n .ok_or_else(|| anyhow!(\"section '{}' not found\", selector))?\n } else {\n content\n };\n\n // Token/char budget truncation\n let content = if let Some(max_tokens) = cmd.max_tokens {\n let max_chars = max_tokens * 4; // rough estimate: 4 chars per token\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if let Some(max_chars) = cmd.max_chars {\n let (truncated, notice) = truncate_at_boundary(&content, max_chars, cmd.offset);\n if let Some(notice) = notice {\n format!(\"{truncated}{notice}\")\n } else {\n truncated\n }\n } else if cmd.offset > 0 {\n if cmd.offset >= content.len() {\n String::new()\n } else {\n content[cmd.offset..].to_string()\n }\n } else {\n content\n };\n\n // Build final output\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = if input.exists() {\n fs::metadata(input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339)\n } else {\n None\n };\n\n let final_output = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: source_label.to_string(),\n output_path: cmd.output.as_ref().map(|p| p.display().to_string()).unwrap_or_else(|| \"-\".to_string()),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content.clone()\n };\n\n // Store in cache (for file inputs only, not URLs/clipboard)\n if !cmd.no_cache && input.exists() && cmd.section.is_none() && cmd.max_tokens.is_none() && cmd.max_chars.is_none() && cmd.offset == 0 {\n if let Ok(key) = cache_key(input, cmd.meta, cmd.raw, cmd.vlm, cmd.ocr, &cmd.section, &cmd.pages) {\n let _ = cache_put(&key, &final_output);\n }\n }\n\n // If VLM already streamed to file, skip normal output\n if converted.already_written {\n return Ok(Ok(()));\n }\n\n Ok(output_final(ctx, cmd, final_output, source_label))\n}\n\n/// Output the final converted content to stdout, file, or JSON/YAML.\nfn output_final(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n content: String,\n source_label: &str,\n) -> Result<()> {\n if let Some(ref output_path) = cmd.output {\n let output_path = expand_path(output_path.clone())?;\n if ctx.common.dry_run {\n info!(\"dry-run: would write to {}\", output_path.display());\n } else {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &content)?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", source_label, output_path.display());\n }\n }\n } else if ctx.common.json {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_json::to_string_pretty(&result)?);\n } else if ctx.common.yaml {\n let result = serde_json::json!({\n \"source\": source_label,\n \"tokens\": estimate_tokens(&content),\n \"content\": content,\n });\n println!(\"{}\", serde_yaml::to_string(&result)?);\n } else {\n print!(\"{}\", content);\n }\n Ok(())\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let result = convert_single_input(ctx, cmd, settings, &temp_file, \"stdin\");\n let _ = fs::remove_file(&temp_file);\n result?\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n info!(\"found {} files to convert\", total);\n\n if ctx.common.dry_run {\n for file in &files {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(ref out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_model.clone(), cmd.vlm_prompt.clone(), cmd.jobs, cmd.output.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n // Apply cleaning unless --raw\n let content = if cmd.raw {\n doc.text_content.clone()\n } else {\n clean_markdown(&doc.text_content)\n };\n\n // Build output: frontmatter only if --meta\n let markdown = if cmd.meta {\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n render_frontmatter_markdown(&frontmatter, &content)?\n } else {\n content\n };\n\n // Write output file if output_dir is specified\n if output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None, None, 1, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Tesseract);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n\n // ===== New feature tests =====\n\n #[test]\n fn clean_markdown_removes_page_numbers() {\n let input = \"Some content\\n\\nPage 3 of 12\\n\\nMore content\\n\\n5\\n\\nEnd\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"Page 3 of 12\"));\n assert!(!cleaned.contains(\"\\n5\\n\"));\n assert!(cleaned.contains(\"Some content\"));\n assert!(cleaned.contains(\"More content\"));\n }\n\n #[test]\n fn clean_markdown_collapses_blank_lines() {\n let input = \"A\\n\\n\\n\\n\\n\\nB\";\n let cleaned = clean_markdown(input);\n assert!(!cleaned.contains(\"\\n\\n\\n\\n\"));\n assert!(cleaned.contains(\"A\"));\n assert!(cleaned.contains(\"B\"));\n }\n\n #[test]\n fn clean_markdown_joins_broken_lines() {\n let input = \"The results\\nshow that our\\nrevenue grew.\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"The results show that our revenue grew.\"));\n }\n\n #[test]\n fn clean_markdown_preserves_headings() {\n let input = \"# Title\\n\\nParagraph\\n\\n## Subtitle\\n\\nMore text\";\n let cleaned = clean_markdown(input);\n assert!(cleaned.contains(\"# Title\"));\n assert!(cleaned.contains(\"## Subtitle\"));\n }\n\n #[test]\n fn extract_toc_basic() {\n let content = \"# Intro\\nSome text\\n## Sub\\nMore text\\n# End\\nFinal\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert_eq!(toc[0].number, \"1\");\n assert_eq!(toc[0].title, \"Intro\");\n assert_eq!(toc[0].level, 1);\n assert_eq!(toc[1].number, \"1.1\");\n assert_eq!(toc[1].title, \"Sub\");\n assert_eq!(toc[2].number, \"2\");\n assert_eq!(toc[2].title, \"End\");\n }\n\n #[test]\n fn extract_section_by_number() {\n let content = \"# First\\nContent A\\n# Second\\nContent B\\n# Third\\nContent C\";\n let section = extract_section(content, \"2\");\n assert!(section.is_some());\n let text = section.unwrap();\n assert!(text.contains(\"# Second\"));\n assert!(text.contains(\"Content B\"));\n assert!(!text.contains(\"Content A\"));\n assert!(!text.contains(\"Content C\"));\n }\n\n #[test]\n fn extract_section_by_name() {\n let content = \"# Introduction\\nHello\\n# Methods\\nWorld\\n# Results\\nDone\";\n let section = extract_section(content, \"methods\");\n assert!(section.is_some());\n assert!(section.unwrap().contains(\"World\"));\n }\n\n #[test]\n fn extract_section_not_found() {\n let content = \"# Title\\nContent\";\n assert!(extract_section(content, \"nonexistent\").is_none());\n }\n\n #[test]\n fn truncate_at_boundary_within_budget() {\n let content = \"Short text\";\n let (result, notice) = truncate_at_boundary(content, 100, 0);\n assert_eq!(result, \"Short text\");\n assert!(notice.is_none());\n }\n\n #[test]\n fn truncate_at_boundary_cuts() {\n let content = \"# First\\nAAA\\n\\n# Second\\nBBB\\n\\n# Third\\nCCC\";\n let (result, notice) = truncate_at_boundary(content, 20, 0);\n assert!(result.len() <= 20);\n assert!(notice.is_some());\n assert!(notice.unwrap().contains(\"truncated\"));\n }\n\n #[test]\n fn parse_page_range_single() {\n let pages = parse_page_range(\"5\").unwrap();\n assert_eq!(pages, vec![5]);\n }\n\n #[test]\n fn parse_page_range_range() {\n let pages = parse_page_range(\"2-5\").unwrap();\n assert_eq!(pages, vec![2, 3, 4, 5]);\n }\n\n #[test]\n fn parse_page_range_complex() {\n let pages = parse_page_range(\"1-3,7,10-12\").unwrap();\n assert_eq!(pages, vec![1, 2, 3, 7, 10, 11, 12]);\n }\n\n #[test]\n fn parse_page_range_zero_rejected() {\n assert!(parse_page_range(\"0\").is_err());\n }\n\n #[test]\n fn parse_page_range_invalid_rejected() {\n assert!(parse_page_range(\"abc\").is_err());\n }\n\n #[test]\n fn estimate_tokens_basic() {\n // ~4 chars per token\n let text = \"Hello world, this is a test sentence.\";\n let tokens = estimate_tokens(text);\n assert!(tokens > 0);\n assert!(tokens < text.len()); // Should be less than char count\n }\n\n #[test]\n fn is_url_detection() {\n assert!(is_url(\"https://example.com\"));\n assert!(is_url(\"http://example.com/path\"));\n assert!(!is_url(\"/local/path\"));\n assert!(!is_url(\"file.pdf\"));\n assert!(!is_url(\"service\"));\n }\n\n #[test]\n fn filter_pages_no_breaks() {\n let content = \"All in one page without breaks\";\n let result = filter_pages(content, &[1]);\n assert!(result.contains(\"All in one page\"));\n }\n\n #[test]\n fn filter_pages_with_formfeed() {\n let content = \"Page one content\\x0cPage two content\\x0cPage three content\";\n let result = filter_pages(content, &[2]);\n assert!(result.contains(\"Page two content\"));\n assert!(!result.contains(\"Page one\"));\n assert!(!result.contains(\"Page three\"));\n }\n\n #[test]\n fn filter_pages_range() {\n let content = \"Page 1\\x0cPage 2\\x0cPage 3\\x0cPage 4\";\n let result = filter_pages(content, &[1, 3]);\n assert!(result.contains(\"Page 1\"));\n assert!(result.contains(\"Page 3\"));\n assert!(!result.contains(\"Page 2\"));\n assert!(!result.contains(\"Page 4\"));\n }\n\n #[test]\n fn toc_detects_tables_and_code() {\n let content = \"# Section A\\nNo special content\\n# Section B\\n| col1 | col2 |\\n|---|---|\\n| a | b |\\n# Section C\\n```rust\\nfn main() {}\\n```\\n\";\n let toc = extract_toc(content);\n assert_eq!(toc.len(), 3);\n assert!(!toc[0].has_table);\n assert!(!toc[0].has_code);\n assert!(toc[1].has_table);\n assert!(!toc[1].has_code);\n assert!(!toc[2].has_table);\n assert!(toc[2].has_code);\n }\n}\n"}, "30051c73a67de94929784cda759ddf13fbfed42d": {"t": 1785757410.5357459, "n": 87283, "h": "23f9fa105b4a00f0", "c": "use std::collections::HashMap;\nuse std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Read, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, AtomicUsize, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, SystemTime};\n\nuse anyhow::{Context, Result, anyhow, bail};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, error, info, warn};\nuse markitdown::{MarkItDown, model::ConversionOptions};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse rayon::prelude::*;\nuse serde::{Deserialize, Serialize};\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\nfn try_main() -> Result<()> {\n let cli = Cli::parse();\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Convert(cmd) => handle_convert(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", alias = \"force\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Convert a single file to Markdown\n Convert(ConvertCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ConvertCommand {\n /// File or directory to convert (use '-' or omit for stdin)\n #[arg(value_name = \"INPUT\")]\n input: Option<PathBuf>,\n /// Write output to a file or directory instead of stdout\n #[arg(short, long, value_name = \"PATH\")]\n output: Option<PathBuf>,\n /// Input format hint when reading from stdin\n #[arg(long, value_name = \"FORMAT\", value_enum)]\n from: Option<InputFormat>,\n /// Recursively process directories\n #[arg(short, long)]\n recursive: bool,\n /// File extensions to process (comma-separated, e.g., \"pdf,docx,html\")\n #[arg(long, value_name = \"EXTENSIONS\", value_delimiter = ',')]\n extensions: Option<Vec<String>>,\n /// Enable VLM processing for images\n #[arg(long)]\n vlm: bool,\n /// Custom VLM prompt for image description\n #[arg(long, value_name = \"PROMPT\")]\n vlm_prompt: Option<String>,\n /// Enable OCR processing for scanned documents\n #[arg(long)]\n ocr: bool,\n /// OCR backend to use\n #[arg(long, value_name = \"BACKEND\", value_enum, default_value = \"tesseract\")]\n ocr_backend: OcrBackend,\n /// OCR languages (comma-separated, e.g., \"eng,deu\")\n #[arg(long, value_name = \"LANGS\", value_delimiter = ',')]\n ocr_languages: Option<Vec<String>>,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, Default)]\nenum InputFormat {\n #[default]\n Auto,\n Html,\n Text,\n Pdf,\n Docx,\n Xlsx,\n Pptx,\n Csv,\n Json,\n Xml,\n Markdown,\n}\n\nimpl InputFormat {\n fn extension(&self) -> Option<&'static str> {\n match self {\n InputFormat::Auto => None,\n InputFormat::Html => Some(\"html\"),\n InputFormat::Text => Some(\"txt\"),\n InputFormat::Pdf => Some(\"pdf\"),\n InputFormat::Docx => Some(\"docx\"),\n InputFormat::Xlsx => Some(\"xlsx\"),\n InputFormat::Pptx => Some(\"pptx\"),\n InputFormat::Csv => Some(\"csv\"),\n InputFormat::Json => Some(\"json\"),\n InputFormat::Xml => Some(\"xml\"),\n InputFormat::Markdown => Some(\"md\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Default, Serialize, Deserialize)]\n#[serde(rename_all = \"lowercase\")]\nenum OcrBackend {\n #[default]\n Tesseract,\n Surya,\n Easyocr,\n}\n\nimpl std::fmt::Display for OcrBackend {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n OcrBackend::Tesseract => write!(f, \"tesseract\"),\n OcrBackend::Surya => write!(f, \"surya\"),\n OcrBackend::Easyocr => write!(f, \"easyocr\"),\n }\n }\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n processors: self.config.processors.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n processors: ProcessorsConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n processors: ProcessorsConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\n/// Processor pipeline configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct ProcessorsConfig {\n /// Ordered list of processors to try\n pipeline: Vec<String>,\n /// File type routing rules (extension -> processor list)\n routing: HashMap<String, Vec<String>>,\n /// VLM processor configuration\n vlm: VlmConfig,\n /// OCR processor configuration\n ocr: OcrConfig,\n}\n\nimpl Default for ProcessorsConfig {\n fn default() -> Self {\n Self {\n pipeline: vec![\"markitdown\".to_string()],\n routing: HashMap::new(),\n vlm: VlmConfig::default(),\n ocr: OcrConfig::default(),\n }\n }\n}\n\n/// VLM (Vision Language Model) processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct VlmConfig {\n /// Enable VLM processing\n enabled: bool,\n /// EAVS server URL\n eavs_url: String,\n /// Provider to use (ollama, openai, anthropic)\n provider: String,\n /// Model name\n model: String,\n /// Custom prompts per file type\n prompts: HashMap<String, String>,\n}\n\nimpl Default for VlmConfig {\n fn default() -> Self {\n let mut prompts = HashMap::new();\n prompts.insert(\n \"default\".to_string(),\n \"Describe this image in detail, including any text visible.\".to_string(),\n );\n prompts.insert(\n \"diagram\".to_string(),\n \"Describe this diagram, including its structure, labels, and relationships.\".to_string(),\n );\n prompts.insert(\n \"screenshot\".to_string(),\n \"Describe this screenshot, including the UI elements and any visible text.\".to_string(),\n );\n\n Self {\n enabled: false,\n eavs_url: \"http://localhost:3000\".to_string(),\n provider: \"ollama\".to_string(),\n model: \"llava\".to_string(),\n prompts,\n }\n }\n}\n\n/// OCR processor configuration\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OcrConfig {\n /// Enable OCR processing\n enabled: bool,\n /// OCR backend (tesseract, surya, easyocr)\n backend: OcrBackend,\n /// Languages for OCR\n languages: Vec<String>,\n}\n\nimpl Default for OcrConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n backend: OcrBackend::Tesseract,\n languages: vec![\"eng\".to_string()],\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or(hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\nstruct ConvertResult {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n markdown: String,\n}\n\n#[derive(Debug, Clone)]\nstruct ConvertedDocument {\n title: Option<String>,\n text_content: String,\n}\n\n#[cfg(test)]\nfn convert_single_file(\n markitdown: &MarkItDown,\n input: &Path,\n conversion_opts: Option<ConversionOptions>,\n) -> Result<ConvertedDocument> {\n let path_str = input\n .to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", input.display()))?;\n\n if let Some(converted) = markitdown.convert(path_str, conversion_opts) {\n return Ok(ConvertedDocument {\n title: converted.title,\n text_content: converted.text_content,\n });\n }\n\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n })\n}\n\nfn render_frontmatter_markdown(frontmatter: &Frontmatter, text_content: &str) -> Result<String> {\n let yaml = serde_yaml::to_string(frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(text_content);\n Ok(body)\n}\n\n/// Statistics for batch conversion\n#[derive(Debug, Clone, Default, Serialize)]\nstruct ConvertStats {\n total: usize,\n converted: usize,\n skipped: usize,\n failed: usize,\n errors: Vec<String>,\n}\n\n/// Processor that handles document conversion with fallback chains\nstruct DocumentProcessor {\n markitdown: MarkItDown,\n settings: ServiceSettings,\n vlm_enabled: bool,\n vlm_prompt: Option<String>,\n ocr_enabled: bool,\n ocr_backend: OcrBackend,\n ocr_languages: Vec<String>,\n}\n\nimpl DocumentProcessor {\n fn new(settings: ServiceSettings) -> Self {\n ConversionService::configure_llm_env(&settings);\n Self {\n markitdown: MarkItDown::new(),\n settings,\n vlm_enabled: false,\n vlm_prompt: None,\n ocr_enabled: false,\n ocr_backend: OcrBackend::Tesseract,\n ocr_languages: vec![\"eng\".to_string()],\n }\n }\n\n fn with_vlm(mut self, enabled: bool, prompt: Option<String>) -> Self {\n self.vlm_enabled = enabled || self.settings.processors.vlm.enabled;\n self.vlm_prompt = prompt;\n self\n }\n\n fn with_ocr(mut self, enabled: bool, backend: OcrBackend, languages: Option<Vec<String>>) -> Self {\n self.ocr_enabled = enabled || self.settings.processors.ocr.enabled;\n if enabled {\n self.ocr_backend = backend;\n } else {\n self.ocr_backend = self.settings.processors.ocr.backend;\n }\n self.ocr_languages = languages.unwrap_or_else(|| self.settings.processors.ocr.languages.clone());\n self\n }\n\n fn process(&self, input: &Path) -> Result<ConvertedDocument> {\n let extension = input\n .extension()\n .and_then(|e| e.to_str())\n .map(|s| s.to_lowercase())\n .unwrap_or_default();\n\n // Check if this is an image and VLM is enabled\n if self.vlm_enabled && is_image_extension(&extension) {\n return self.process_with_vlm(input, &extension);\n }\n\n // Try markitdown first\n let conversion_opts = self.build_conversion_options();\n match self.markitdown.convert(\n input.to_str().ok_or_else(|| anyhow!(\"invalid path encoding\"))?,\n conversion_opts,\n ) {\n Some(result) if !result.text_content.trim().is_empty() => {\n return Ok(ConvertedDocument {\n title: result.title,\n text_content: result.text_content,\n });\n }\n _ => {}\n }\n\n // If OCR is enabled and we got empty content, try OCR\n if self.ocr_enabled\n && (is_pdf_extension(&extension) || is_image_extension(&extension))\n && let Ok(ocr_result) = self.process_with_ocr(input)\n && !ocr_result.text_content.trim().is_empty()\n {\n return Ok(ocr_result);\n }\n\n // Fallback: try reading as text\n let text = fs::read_to_string(input).with_context(|| {\n format!(\n \"no converter available for {}, and file could not be read as UTF-8 text\",\n input.display()\n )\n })?;\n\n Ok(ConvertedDocument {\n title: None,\n text_content: text,\n })\n }\n\n fn build_conversion_options(&self) -> Option<ConversionOptions> {\n if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n }\n }\n\n fn process_with_vlm(&self, input: &Path, _extension: &str) -> Result<ConvertedDocument> {\n let vlm_config = &self.settings.processors.vlm;\n let prompt = self.vlm_prompt.as_ref()\n .or(vlm_config.prompts.get(\"default\"))\n .map(|s| s.as_str())\n .unwrap_or(\"Describe this image in detail.\");\n\n // Read and encode image as base64\n let image_data = fs::read(input)\n .with_context(|| format!(\"reading image file {}\", input.display()))?;\n let base64_image = base64_encode(&image_data);\n\n // Determine MIME type\n let extension = input.extension()\n .and_then(|e| e.to_str())\n .unwrap_or(\"png\");\n let mime_type = match extension.to_lowercase().as_str() {\n \"jpg\" | \"jpeg\" => \"image/jpeg\",\n \"png\" => \"image/png\",\n \"gif\" => \"image/gif\",\n \"webp\" => \"image/webp\",\n \"bmp\" => \"image/bmp\",\n _ => \"image/png\",\n };\n\n // Call EAVS for VLM processing\n let description = call_eavs_vlm(\n &vlm_config.eavs_url,\n &vlm_config.provider,\n &vlm_config.model,\n &base64_image,\n mime_type,\n prompt,\n )?;\n\n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: format!(\"# Image: {}\\n\\n{}\", \n input.file_name().and_then(|s| s.to_str()).unwrap_or(\"image\"),\n description\n ),\n })\n }\n\n fn process_with_ocr(&self, input: &Path) -> Result<ConvertedDocument> {\n let text = run_ocr(input, self.ocr_backend, &self.ocr_languages)?;\n \n let title = input.file_stem()\n .and_then(|s| s.to_str())\n .map(|s| s.to_string());\n\n Ok(ConvertedDocument {\n title,\n text_content: text,\n })\n }\n}\n\nfn is_image_extension(ext: &str) -> bool {\n matches!(ext, \"jpg\" | \"jpeg\" | \"png\" | \"gif\" | \"webp\" | \"bmp\" | \"tiff\" | \"tif\")\n}\n\nfn is_pdf_extension(ext: &str) -> bool {\n ext == \"pdf\"\n}\n\nfn base64_encode(data: &[u8]) -> String {\n const ALPHABET: &[u8] = b\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n let mut result = String::with_capacity(data.len().div_ceil(3) * 4);\n \n for chunk in data.chunks(3) {\n let b0 = chunk[0] as usize;\n let b1 = chunk.get(1).copied().unwrap_or(0) as usize;\n let b2 = chunk.get(2).copied().unwrap_or(0) as usize;\n \n result.push(ALPHABET[b0 >> 2] as char);\n result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char);\n \n if chunk.len() > 1 {\n result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char);\n } else {\n result.push('=');\n }\n \n if chunk.len() > 2 {\n result.push(ALPHABET[b2 & 0x3f] as char);\n } else {\n result.push('=');\n }\n }\n \n result\n}\n\n/// Call EAVS server for VLM processing\nfn call_eavs_vlm(\n eavs_url: &str,\n provider: &str,\n model: &str,\n base64_image: &str,\n mime_type: &str,\n prompt: &str,\n) -> Result<String> {\n let client = reqwest::blocking::Client::new();\n \n let request_body = serde_json::json!({\n \"model\": model,\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": prompt\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": format!(\"data:{};base64,{}\", mime_type, base64_image)\n }\n }\n ]\n }],\n \"max_tokens\": 4096\n });\n\n let response = client\n .post(format!(\"{}/v1/chat/completions\", eavs_url))\n .header(\"Content-Type\", \"application/json\")\n .header(\"X-Provider\", provider)\n .json(&request_body)\n .send()\n .context(\"sending request to EAVS\")?;\n\n if !response.status().is_success() {\n let status = response.status();\n let body = response.text().unwrap_or_default();\n bail!(\"EAVS request failed with status {}: {}\", status, body);\n }\n\n let response_json: serde_json::Value = response.json()\n .context(\"parsing EAVS response\")?;\n\n response_json[\"choices\"][0][\"message\"][\"content\"]\n .as_str()\n .map(|s| s.to_string())\n .ok_or_else(|| anyhow!(\"invalid response format from EAVS\"))\n}\n\n/// Run OCR on a file\nfn run_ocr(input: &Path, backend: OcrBackend, languages: &[String]) -> Result<String> {\n let lang_arg = languages.join(\"+\");\n \n match backend {\n OcrBackend::Tesseract => {\n let output = ProcCommand::new(\"tesseract\")\n .arg(input)\n .arg(\"stdout\")\n .arg(\"-l\")\n .arg(&lang_arg)\n .output()\n .context(\"running tesseract OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"tesseract failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Surya => {\n // Surya uses Python, call via python\n let output = ProcCommand::new(\"surya_ocr\")\n .arg(input)\n .arg(\"--langs\")\n .arg(&lang_arg)\n .output()\n .context(\"running surya OCR\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"surya failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n OcrBackend::Easyocr => {\n // EasyOCR via Python command\n let script = format!(\n r#\"import easyocr; import sys; reader = easyocr.Reader(['{}']); result = reader.readtext('{}'); print('\\n'.join([text for _, text, _ in result]))\"#,\n lang_arg.replace(\"+\", \"','\"),\n input.display()\n );\n \n let output = ProcCommand::new(\"python3\")\n .arg(\"-c\")\n .arg(&script)\n .output()\n .context(\"running easyocr\")?;\n\n if !output.status.success() {\n let stderr = String::from_utf8_lossy(&output.stderr);\n bail!(\"easyocr failed: {}\", stderr);\n }\n\n Ok(String::from_utf8_lossy(&output.stdout).to_string())\n }\n }\n}\n\nfn handle_convert(ctx: &RuntimeContext, cmd: ConvertCommand) -> Result<()> {\n let mut settings = ctx.service_settings(&ServiceRunOpts {\n watch_dir: None,\n output_dir: None,\n index_dir: None,\n disable_index: true,\n once: true,\n })?;\n settings.index_enabled = false;\n\n // Check if reading from stdin\n let is_stdin = cmd.input.is_none() \n || cmd.input.as_ref().map(|p| p.as_os_str() == \"-\").unwrap_or(false);\n\n if is_stdin {\n return handle_convert_stdin(ctx, &cmd, &settings);\n }\n\n let input = expand_path(cmd.input.clone().unwrap())?;\n\n // Check if input is a directory\n if input.is_dir() {\n return handle_convert_directory(ctx, &cmd, &settings, &input);\n }\n\n // Single file conversion\n if !input.is_file() {\n bail!(\"input path is not a file or directory: {}\", input.display());\n }\n\n let processor = DocumentProcessor::new(settings)\n .with_vlm(cmd.vlm, cmd.vlm_prompt.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(&input)?;\n\n let output = cmd\n .output\n .map(expand_path)\n .transpose()\n .context(\"expanding output path\")?;\n\n let output_path_str = output\n .as_ref()\n .map(|p| p.display().to_string())\n .unwrap_or_else(|| \"-\".to_string());\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(&input)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: output_path_str.clone(),\n source_modified: source_modified.clone(),\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n\n output_convert_result(ctx, output, &ConvertResult {\n source_path: input.display().to_string(),\n output_path: output_path_str,\n source_modified,\n title: converted.title,\n converted_at,\n markdown,\n })\n}\n\nfn handle_convert_stdin(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n) -> Result<()> {\n let mut content = Vec::new();\n io::stdin().read_to_end(&mut content)\n .context(\"reading from stdin\")?;\n\n // Determine file extension from format hint\n let extension = cmd.from.unwrap_or(InputFormat::Auto).extension();\n \n // Create a temporary file with the appropriate extension\n let temp_dir = std::env::temp_dir();\n let temp_file = if let Some(ext) = extension {\n temp_dir.join(format!(\"ingestr-stdin.{}\", ext))\n } else {\n // Try to auto-detect format from content\n let detected_ext = detect_format_from_content(&content);\n temp_dir.join(format!(\"ingestr-stdin.{}\", detected_ext))\n };\n\n fs::write(&temp_file, &content)\n .context(\"writing stdin content to temp file\")?;\n\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_prompt.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n let converted = processor.process(&temp_file);\n \n // Clean up temp file\n let _ = fs::remove_file(&temp_file);\n\n let converted = converted?;\n\n let output = cmd\n .output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n let output_path_str = output\n .as_ref()\n .map(|p| p.display().to_string())\n .unwrap_or_else(|| \"-\".to_string());\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let frontmatter = Frontmatter {\n source_path: \"stdin\".to_string(),\n output_path: output_path_str.clone(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n\n output_convert_result(ctx, output, &ConvertResult {\n source_path: \"stdin\".to_string(),\n output_path: output_path_str,\n source_modified: None,\n title: converted.title,\n converted_at,\n markdown,\n })\n}\n\nfn detect_format_from_content(content: &[u8]) -> &'static str {\n // Check for common file signatures\n if content.starts_with(b\"%PDF\") {\n return \"pdf\";\n }\n if content.starts_with(b\"PK\\x03\\x04\") {\n // Could be docx, xlsx, pptx - default to docx\n return \"docx\";\n }\n if content.starts_with(b\"<!DOCTYPE html\") || content.starts_with(b\"<html\") || content.starts_with(b\"<HTML\") {\n return \"html\";\n }\n if content.starts_with(b\"<?xml\") {\n return \"xml\";\n }\n if content.starts_with(b\"{\") || content.starts_with(b\"[\") {\n return \"json\";\n }\n \n // Default to text\n \"txt\"\n}\n\nfn handle_convert_directory(\n ctx: &RuntimeContext,\n cmd: &ConvertCommand,\n settings: &ServiceSettings,\n input_dir: &Path,\n) -> Result<()> {\n let output_dir = cmd.output\n .as_ref()\n .map(|p| expand_path(p.clone()))\n .transpose()\n .context(\"expanding output path\")?;\n\n // Collect files to process\n let walker = if cmd.recursive {\n WalkDir::new(input_dir)\n } else {\n WalkDir::new(input_dir).max_depth(1)\n };\n\n let extensions: Option<Vec<String>> = cmd.extensions.clone()\n .map(|exts| exts.iter().map(|e| e.to_lowercase()).collect());\n\n let files: Vec<PathBuf> = walker\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .filter(|e| {\n // Skip hidden files if skip_hidden is set\n if settings.skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n if files.is_empty() {\n info!(\"no files found to convert\");\n return Ok(());\n }\n\n let total = files.len();\n info!(\"found {} files to convert\", total);\n\n if ctx.common.dry_run {\n for file in &files {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(ref out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out.display().to_string()\n } else {\n \"-\".to_string()\n };\n println!(\"Would convert {} -> {}\", file.display(), output_path);\n }\n return Ok(());\n }\n\n // Use parallel processing if requested\n let parallel = ctx.common.parallel.unwrap_or_else(|| {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n });\n\n let converted = AtomicUsize::new(0);\n let failed = AtomicUsize::new(0);\n let errors: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());\n\n let results: Vec<Result<ConvertResult>> = if parallel > 1 {\n rayon::ThreadPoolBuilder::new()\n .num_threads(parallel)\n .build()\n .context(\"building thread pool\")?\n .install(|| {\n files.par_iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n })\n } else {\n files.iter().map(|file| {\n process_file_for_batch(\n file,\n input_dir,\n output_dir.as_ref(),\n settings,\n cmd,\n &converted,\n &failed,\n &errors,\n )\n }).collect()\n };\n\n let stats = ConvertStats {\n total,\n converted: converted.load(Ordering::Relaxed),\n skipped: 0,\n failed: failed.load(Ordering::Relaxed),\n errors: errors.into_inner().unwrap_or_default(),\n };\n\n // Output results\n if ctx.common.json {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_json::to_string_pretty(&output)?);\n } else if ctx.common.yaml {\n let successful_results: Vec<_> = results.into_iter().filter_map(Result::ok).collect();\n let output = serde_json::json!({\n \"stats\": stats,\n \"results\": successful_results,\n });\n println!(\"{}\", serde_yaml::to_string(&output)?);\n } else {\n println!(\"\\nConversion complete:\");\n println!(\" Total files: {}\", stats.total);\n println!(\" Converted: {}\", stats.converted);\n println!(\" Failed: {}\", stats.failed);\n if !stats.errors.is_empty() {\n println!(\"\\nErrors:\");\n for err in &stats.errors {\n println!(\" - {}\", err);\n }\n }\n }\n\n Ok(())\n}\n\n#[allow(clippy::too_many_arguments)]\nfn process_file_for_batch(\n file: &Path,\n input_dir: &Path,\n output_dir: Option<&PathBuf>,\n settings: &ServiceSettings,\n cmd: &ConvertCommand,\n converted: &AtomicUsize,\n failed: &AtomicUsize,\n errors: &std::sync::Mutex<Vec<String>>,\n) -> Result<ConvertResult> {\n let processor = DocumentProcessor::new(settings.clone())\n .with_vlm(cmd.vlm, cmd.vlm_prompt.clone())\n .with_ocr(cmd.ocr, cmd.ocr_backend, cmd.ocr_languages.clone());\n\n match processor.process(file) {\n Ok(doc) => {\n let relative = file.strip_prefix(input_dir).unwrap_or(file);\n let output_path = if let Some(out_dir) = output_dir {\n let mut out = out_dir.join(relative);\n out.set_extension(\"md\");\n out\n } else {\n PathBuf::from(\"-\")\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(file)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified: source_modified.clone(),\n title: doc.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &doc.text_content)?;\n\n // Write output file if output_dir is specified\n if output_dir.is_some() {\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(&output_path, &markdown)?;\n }\n\n converted.fetch_add(1, Ordering::Relaxed);\n info!(\"converted {} -> {}\", file.display(), output_path.display());\n\n Ok(ConvertResult {\n source_path: file.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: doc.title,\n converted_at,\n markdown,\n })\n }\n Err(e) => {\n failed.fetch_add(1, Ordering::Relaxed);\n let err_msg = format!(\"{}: {}\", file.display(), e);\n errors.lock().unwrap().push(err_msg.clone());\n error!(\"failed to convert {}: {}\", file.display(), e);\n Err(e)\n }\n }\n}\n\nfn output_convert_result(\n ctx: &RuntimeContext,\n output: Option<PathBuf>,\n result: &ConvertResult,\n) -> Result<()> {\n if let Some(output) = output {\n if ctx.common.dry_run {\n info!(\"dry-run: would write markdown to {}\", output.display());\n } else {\n if let Some(parent) = output.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n fs::write(&output, &result.markdown)\n .with_context(|| format!(\"writing markdown to {}\", output.display()))?;\n if !ctx.common.quiet {\n println!(\"Converted {} -> {}\", result.source_path, output.display());\n }\n }\n } else if ctx.common.json {\n println!(\"{}\", serde_json::to_string_pretty(result)?);\n } else if ctx.common.yaml {\n println!(\"{}\", serde_yaml::to_string(result)?);\n } else {\n print!(\"{}\", result.markdown);\n }\n Ok(())\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n processors: ProcessorsConfig,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind;\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\")\n && let Some(idx) = rest.find(\"\\n---\\n\")\n {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::time::UNIX_EPOCH;\n\n fn unique_temp_dir() -> PathBuf {\n let nanos = SystemTime::now()\n .duration_since(UNIX_EPOCH)\n .expect(\"time since epoch\")\n .as_nanos();\n std::env::temp_dir().join(format!(\"ingestr-test-{nanos}\"))\n }\n\n fn create_test_settings() -> ServiceSettings {\n ServiceSettings {\n watch_dir: PathBuf::from(\"/tmp\"),\n output_dir: PathBuf::from(\"/tmp/output\"),\n index_dir: PathBuf::from(\"/tmp/index\"),\n index_enabled: false,\n debounce: Duration::from_millis(50),\n skip_hidden: true,\n data_dir: PathBuf::from(\"/tmp/data\"),\n state_dir: PathBuf::from(\"/tmp/state\"),\n llm_enabled: false,\n llm_client: \"openai\".to_string(),\n llm_model: \"gpt-4o\".to_string(),\n llm_base_url: None,\n llm_api_key: None,\n processors: ProcessorsConfig::default(),\n }\n }\n\n #[test]\n fn convert_single_file_renders_frontmatter_and_content() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"sample.txt\");\n fs::write(&input, \"Hello world\")?;\n\n let markitdown = MarkItDown::new();\n let converted = convert_single_file(&markitdown, &input, None)?;\n\n let frontmatter = Frontmatter {\n source_path: input.display().to_string(),\n output_path: \"-\".to_string(),\n source_modified: None,\n title: converted.title.clone(),\n converted_at: \"2025-01-01T00:00:00Z\".to_string(),\n };\n\n let markdown = render_frontmatter_markdown(&frontmatter, &converted.text_content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter parsed\");\n assert_eq!(parsed.source_path, input.display().to_string());\n assert_eq!(parsed.output_path, \"-\");\n assert!(body.contains(\"Hello world\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_converts_text_file() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Test content for processor\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings);\n let result = processor.process(&input)?;\n\n assert!(result.text_content.contains(\"Test content for processor\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_vlm_disabled_skips_vlm() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_vlm(false, None);\n \n assert!(!processor.vlm_enabled);\n \n let result = processor.process(&input)?;\n assert!(result.text_content.contains(\"Plain text\"));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn document_processor_with_ocr_disabled_skips_ocr() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n let input = dir.join(\"test.txt\");\n fs::write(&input, \"Plain text\")?;\n\n let settings = create_test_settings();\n let processor = DocumentProcessor::new(settings)\n .with_ocr(false, OcrBackend::Tesseract, None);\n \n assert!(!processor.ocr_enabled);\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn detect_format_from_content_identifies_pdf() {\n let content = b\"%PDF-1.4 rest of content\";\n assert_eq!(detect_format_from_content(content), \"pdf\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_html() {\n let content = b\"<!DOCTYPE html><html>test</html>\";\n assert_eq!(detect_format_from_content(content), \"html\");\n \n let content2 = b\"<html><body>test</body></html>\";\n assert_eq!(detect_format_from_content(content2), \"html\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_docx() {\n let content = b\"PK\\x03\\x04rest\";\n assert_eq!(detect_format_from_content(content), \"docx\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_json() {\n let content = b\"{\\\"key\\\": \\\"value\\\"}\";\n assert_eq!(detect_format_from_content(content), \"json\");\n \n let content2 = b\"[1, 2, 3]\";\n assert_eq!(detect_format_from_content(content2), \"json\");\n }\n\n #[test]\n fn detect_format_from_content_identifies_xml() {\n let content = b\"<?xml version=\\\"1.0\\\"?><root></root>\";\n assert_eq!(detect_format_from_content(content), \"xml\");\n }\n\n #[test]\n fn detect_format_from_content_defaults_to_txt() {\n let content = b\"Just some plain text\";\n assert_eq!(detect_format_from_content(content), \"txt\");\n }\n\n #[test]\n fn is_image_extension_works() {\n assert!(is_image_extension(\"jpg\"));\n assert!(is_image_extension(\"jpeg\"));\n assert!(is_image_extension(\"png\"));\n assert!(is_image_extension(\"gif\"));\n assert!(is_image_extension(\"webp\"));\n assert!(!is_image_extension(\"pdf\"));\n assert!(!is_image_extension(\"txt\"));\n assert!(!is_image_extension(\"docx\"));\n }\n\n #[test]\n fn is_pdf_extension_works() {\n assert!(is_pdf_extension(\"pdf\"));\n assert!(!is_pdf_extension(\"txt\"));\n assert!(!is_pdf_extension(\"docx\"));\n }\n\n #[test]\n fn base64_encode_works() {\n let data = b\"Hello, World!\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"SGVsbG8sIFdvcmxkIQ==\");\n }\n\n #[test]\n fn base64_encode_empty() {\n let data = b\"\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"\");\n }\n\n #[test]\n fn base64_encode_single_byte() {\n let data = b\"A\";\n let encoded = base64_encode(data);\n assert_eq!(encoded, \"QQ==\");\n }\n\n #[test]\n fn input_format_extension() {\n assert_eq!(InputFormat::Html.extension(), Some(\"html\"));\n assert_eq!(InputFormat::Pdf.extension(), Some(\"pdf\"));\n assert_eq!(InputFormat::Docx.extension(), Some(\"docx\"));\n assert_eq!(InputFormat::Auto.extension(), None);\n }\n\n #[test]\n fn ocr_backend_display() {\n assert_eq!(OcrBackend::Tesseract.to_string(), \"tesseract\");\n assert_eq!(OcrBackend::Surya.to_string(), \"surya\");\n assert_eq!(OcrBackend::Easyocr.to_string(), \"easyocr\");\n }\n\n #[test]\n fn processors_config_default() {\n let config = ProcessorsConfig::default();\n assert_eq!(config.pipeline, vec![\"markitdown\"]);\n assert!(config.routing.is_empty());\n assert!(!config.vlm.enabled);\n assert!(!config.ocr.enabled);\n }\n\n #[test]\n fn vlm_config_default_prompts() {\n let config = VlmConfig::default();\n assert!(config.prompts.contains_key(\"default\"));\n assert!(config.prompts.contains_key(\"diagram\"));\n assert!(config.prompts.contains_key(\"screenshot\"));\n }\n\n #[test]\n fn ocr_config_default() {\n let config = OcrConfig::default();\n assert!(!config.enabled);\n assert_eq!(config.backend, OcrBackend::Tesseract);\n assert_eq!(config.languages, vec![\"eng\"]);\n }\n\n #[test]\n fn convert_stats_default() {\n let stats = ConvertStats::default();\n assert_eq!(stats.total, 0);\n assert_eq!(stats.converted, 0);\n assert_eq!(stats.skipped, 0);\n assert_eq!(stats.failed, 0);\n assert!(stats.errors.is_empty());\n }\n\n #[test]\n fn batch_convert_collects_files_by_extension() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"doc1.txt\"), \"text 1\")?;\n fs::write(dir.join(\"doc2.txt\"), \"text 2\")?;\n fs::write(dir.join(\"doc3.pdf\"), \"pdf content\")?;\n fs::write(dir.join(\"doc4.html\"), \"<html>test</html>\")?;\n\n let extensions = Some(vec![\"txt\".to_string()]);\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if let Some(ref exts) = extensions {\n e.path()\n .extension()\n .and_then(|ext| ext.to_str())\n .map(|ext| exts.contains(&ext.to_lowercase()))\n .unwrap_or(false)\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 2);\n assert!(files.iter().all(|f| f.extension().map(|e| e == \"txt\").unwrap_or(false)));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn batch_convert_skips_hidden_files() -> Result<()> {\n let dir = unique_temp_dir();\n fs::create_dir_all(&dir)?;\n \n // Create test files\n fs::write(dir.join(\"visible.txt\"), \"visible\")?;\n fs::write(dir.join(\".hidden.txt\"), \"hidden\")?;\n\n let skip_hidden = true;\n let files: Vec<PathBuf> = WalkDir::new(&dir)\n .max_depth(1)\n .into_iter()\n .filter_map(Result::ok)\n .filter(|e| e.file_type().is_file())\n .filter(|e| {\n if skip_hidden {\n !e.path()\n .components()\n .any(|c| c.as_os_str().to_str().map(|s| s.starts_with('.')).unwrap_or(false))\n } else {\n true\n }\n })\n .map(|e| e.path().to_path_buf())\n .collect();\n\n assert_eq!(files.len(), 1);\n assert!(files[0].file_name().map(|n| n == \"visible.txt\").unwrap_or(false));\n\n let _ = fs::remove_dir_all(&dir);\n Ok(())\n }\n\n #[test]\n fn frontmatter_serialization_roundtrip() -> Result<()> {\n let frontmatter = Frontmatter {\n source_path: \"/test/input.txt\".to_string(),\n output_path: \"/test/output.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Test Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n };\n\n let content = \"Test content here.\";\n let markdown = render_frontmatter_markdown(&frontmatter, content)?;\n let (parsed, body) = parse_frontmatter(&markdown);\n\n let parsed = parsed.expect(\"frontmatter should parse\");\n assert_eq!(parsed.source_path, frontmatter.source_path);\n assert_eq!(parsed.output_path, frontmatter.output_path);\n assert_eq!(parsed.title, frontmatter.title);\n assert!(body.contains(content));\n\n Ok(())\n }\n\n #[test]\n fn parse_frontmatter_handles_missing() {\n let content = \"No frontmatter here\";\n let (fm, body) = parse_frontmatter(content);\n assert!(fm.is_none());\n assert_eq!(body, content);\n }\n\n #[test]\n fn parse_frontmatter_handles_malformed() {\n let content = \"---\\ninvalid: yaml: here\\n---\\nBody content\";\n let (fm, _body) = parse_frontmatter(content);\n // Should return None for malformed YAML\n assert!(fm.is_none());\n }\n\n #[test]\n fn convert_result_json_serialization() -> Result<()> {\n let result = ConvertResult {\n source_path: \"/test.txt\".to_string(),\n output_path: \"/test.md\".to_string(),\n source_modified: Some(\"2025-01-01T00:00:00Z\".to_string()),\n title: Some(\"Title\".to_string()),\n converted_at: \"2025-01-01T00:00:01Z\".to_string(),\n markdown: \"# Title\\n\\nContent\".to_string(),\n };\n\n let json = serde_json::to_string(&result)?;\n assert!(json.contains(\"source_path\"));\n assert!(json.contains(\"output_path\"));\n assert!(json.contains(\"markdown\"));\n\n let parsed: ConvertResult = serde_json::from_str(&json)?;\n assert_eq!(parsed.source_path, result.source_path);\n\n Ok(())\n }\n}\n"}, "f08c782bfa3b8770235e6b7cc65cd8251ca1a70a": {"t": 1785757412.1684995, "n": 43724, "h": "5a3f0db44934d2e9", "c": "use std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, SystemTime};\n\nuse anyhow::{Context, Result, anyhow};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, info, warn};\nuse markitdown::{model::ConversionOptions, MarkItDown};\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse serde::{Deserialize, Serialize};\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\nfn try_main() -> Result<()> {\n let cli = Cli::parse();\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", alias = \"force\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n llm_enabled: self.config.llm.enabled,\n llm_client: self.config.llm.client.clone(),\n llm_model: self.config.llm.model.clone(),\n llm_base_url: self.config.llm.base_url.clone(),\n llm_api_key: self.config.llm.api_key.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n llm: LlmConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n llm: LlmConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LlmConfig {\n /// Enable LLM-based image description generation\n enabled: bool,\n /// LLM provider: \"openai\", \"gemini\", or \"deepseek\"\n client: String,\n /// Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\n model: String,\n /// Custom API base URL (for OpenAI-compatible endpoints)\n base_url: Option<String>,\n /// API key (optional, can also use environment variables)\n api_key: Option<String>,\n}\n\nimpl Default for LlmConfig {\n fn default() -> Self {\n Self {\n enabled: false,\n client: \"openai\".to_string(),\n model: \"gpt-4o\".to_string(),\n base_url: None,\n api_key: None,\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or_else(|| hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n llm_enabled: bool,\n llm_client: String,\n llm_model: String,\n llm_base_url: Option<String>,\n llm_api_key: Option<String>,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n // Set LLM environment variables from config if provided\n Self::configure_llm_env(&settings);\n\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn configure_llm_env(settings: &ServiceSettings) {\n if !settings.llm_enabled {\n return;\n }\n\n // Set API key environment variable based on provider\n if let Some(ref api_key) = settings.llm_api_key {\n let env_var = match settings.llm_client.as_str() {\n \"openai\" => \"OPENAI_API_KEY\",\n \"gemini\" => \"GEMINI_API_KEY\",\n \"deepseek\" => \"DEEPSEEK_API_KEY\",\n _ => \"OPENAI_API_KEY\",\n };\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(env_var, api_key);\n }\n }\n\n // Set base URL environment variable (OpenAI-compatible format)\n if let Some(ref base_url) = settings.llm_base_url {\n // SAFETY: This is called at service startup before any threads are spawned,\n // and we control the environment variable names being set.\n unsafe {\n env::set_var(\"OPENAI_API_BASE\", base_url);\n env::set_var(\"OPENAI_BASE_URL\", base_url);\n }\n }\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind.clone();\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let conversion_opts = if self.settings.llm_enabled {\n Some(ConversionOptions {\n file_extension: None,\n url: None,\n llm_client: Some(self.settings.llm_client.clone()),\n llm_model: Some(self.settings.llm_model.clone()),\n })\n } else {\n None\n };\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n conversion_opts,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\") {\n if let Some(idx) = rest.find(\"\\n---\\n\") {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n"}, "983c06cc50811c79a4fd476475acbf0808a10bdc": {"t": 1785757413.7117143, "n": 40756, "h": "a43216d4f190f94f", "c": "use std::env;\nuse std::fmt;\nuse std::fs::{self, OpenOptions};\nuse std::io::{self, IsTerminal, Write};\nuse std::path::{Path, PathBuf};\nuse std::process::{Command as ProcCommand, Stdio};\nuse std::sync::{\n Arc,\n atomic::{AtomicBool, Ordering},\n mpsc::{self, RecvTimeoutError},\n};\nuse std::time::{Duration, SystemTime};\n\nuse anyhow::{Context, Result, anyhow};\nuse clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};\nuse clap_complete::Shell;\nuse config::{Config, Environment, File, FileFormat};\nuse env_logger::fmt::WriteStyle;\nuse ingestr_core::{IndexedDocument, SearchIndex};\nuse log::{LevelFilter, debug, info, warn};\nuse markitdown::MarkItDown;\nuse notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse serde::{Deserialize, Serialize};\nuse sysinfo::{Pid, Signal, System};\nuse time::OffsetDateTime;\nuse time::format_description::well_known::Rfc3339;\nuse walkdir::WalkDir;\n\nconst APP_NAME: &str = env!(\"CARGO_PKG_NAME\");\nconst CONFIG_DIR_NAME: &str = \"ingestr\";\n\nfn main() {\n if let Err(err) = try_main() {\n let _ = writeln!(io::stderr(), \"{err:?}\");\n std::process::exit(1);\n }\n}\n\nfn try_main() -> Result<()> {\n let cli = Cli::parse();\n\n let mut ctx = RuntimeContext::new(cli.common.clone())?;\n ctx.init_logging()?;\n debug!(\"resolved paths: {:#?}\", ctx.paths);\n\n match cli.command {\n Command::Service { command } => handle_service(&mut ctx, command),\n Command::Search(cmd) => handle_search(&ctx, cmd),\n Command::Init(cmd) => handle_init(&ctx, cmd),\n Command::Config { command } => handle_config(&ctx, command),\n Command::Completions { shell } => handle_completions(shell),\n }\n}\n\n#[derive(Debug, Parser)]\n#[command(\n author,\n version,\n about = \"Background service that converts documents to Markdown and indexes them for search.\",\n propagate_version = true\n)]\nstruct Cli {\n #[command(flatten)]\n common: CommonOpts,\n #[command(subcommand)]\n command: Command,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct CommonOpts {\n /// Override the config file path\n #[arg(long, value_name = \"PATH\", global = true)]\n config: Option<PathBuf>,\n /// Reduce output to only errors\n #[arg(short, long, action = clap::ArgAction::SetTrue, global = true)]\n quiet: bool,\n /// Increase logging verbosity (stackable)\n #[arg(short = 'v', long = \"verbose\", action = clap::ArgAction::Count, global = true)]\n verbose: u8,\n /// Enable debug logging (equivalent to -vv)\n #[arg(long, global = true)]\n debug: bool,\n /// Enable trace logging (overrides other levels)\n #[arg(long, global = true)]\n trace: bool,\n /// Output machine readable JSON\n #[arg(long, global = true, conflicts_with = \"yaml\")]\n json: bool,\n /// Output machine readable YAML\n #[arg(long, global = true)]\n yaml: bool,\n /// Disable ANSI colors in output\n #[arg(long = \"no-color\", global = true, conflicts_with = \"color\")]\n no_color: bool,\n /// Control color output (auto, always, never)\n #[arg(long, value_enum, default_value_t = ColorOption::Auto, global = true)]\n color: ColorOption,\n /// Do not change anything on disk\n #[arg(long = \"dry-run\", global = true)]\n dry_run: bool,\n /// Assume \"yes\" for interactive prompts\n #[arg(short = 'y', long = \"yes\", alias = \"force\", global = true)]\n assume_yes: bool,\n /// Maximum seconds to allow an operation to run\n #[arg(long = \"timeout\", value_name = \"SECONDS\", global = true)]\n timeout: Option<u64>,\n /// Override the degree of parallelism\n #[arg(long = \"parallel\", value_name = \"N\", global = true)]\n parallel: Option<usize>,\n /// Disable progress indicators\n #[arg(long = \"no-progress\", global = true)]\n no_progress: bool,\n /// Emit additional diagnostics for troubleshooting\n #[arg(long, global = true)]\n diagnostics: bool,\n}\n\n#[derive(Debug, Clone, Copy, ValueEnum)]\nenum ColorOption {\n Auto,\n Always,\n Never,\n}\n\n#[derive(Debug, Subcommand)]\nenum Command {\n /// Manage the background conversion and indexing service\n Service {\n #[command(subcommand)]\n command: ServiceCommand,\n },\n /// Query the search index\n Search(SearchCommand),\n /// Create config directories and default files\n Init(InitCommand),\n /// Inspect and manage configuration\n Config {\n #[command(subcommand)]\n command: ConfigCommand,\n },\n /// Generate shell completions\n Completions {\n #[arg(value_enum)]\n shell: Shell,\n },\n}\n\n#[derive(Debug, Clone, Args)]\nstruct ServiceRunOpts {\n /// Directory to watch for new or updated documents\n #[arg(long, value_name = \"PATH\")]\n watch_dir: Option<PathBuf>,\n /// Directory to write converted markdown files\n #[arg(long, value_name = \"PATH\")]\n output_dir: Option<PathBuf>,\n /// Directory to store the search index\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n /// Disable indexing while still converting files\n #[arg(long)]\n disable_index: bool,\n /// Convert the current contents and exit instead of watching for changes\n #[arg(long)]\n once: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ServiceCommand {\n /// Run the service in the foreground\n Run(ServiceRunOpts),\n /// Start the service in the background\n Start(ServiceRunOpts),\n /// Stop the background service\n Stop,\n /// Restart the background service\n Restart(ServiceRunOpts),\n /// Show service status\n Status,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct SearchCommand {\n /// Search query\n query: String,\n /// Maximum results to return\n #[arg(long, default_value_t = 10)]\n limit: usize,\n /// Override the index directory\n #[arg(long, value_name = \"PATH\")]\n index_dir: Option<PathBuf>,\n}\n\n#[derive(Debug, Clone, Args)]\nstruct InitCommand {\n /// Recreate configuration even if it already exists\n #[arg(long = \"force\")]\n force: bool,\n}\n\n#[derive(Debug, Subcommand)]\nenum ConfigCommand {\n /// Output the effective configuration\n Show,\n /// Print the resolved config file path\n Path,\n /// Regenerate the default configuration file\n Reset,\n}\n\n#[derive(Debug, Clone)]\nstruct RuntimeContext {\n common: CommonOpts,\n paths: AppPaths,\n config: AppConfig,\n directories: ResolvedDirectories,\n}\n\n#[derive(Debug, Clone)]\nstruct ResolvedDirectories {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n}\n\nimpl RuntimeContext {\n fn new(common: CommonOpts) -> Result<Self> {\n let mut paths = AppPaths::discover(common.config.clone())?;\n let config = load_or_init_config(&mut paths, &common)?;\n let paths = paths.apply_overrides(&config)?;\n let directories = ResolvedDirectories::from_config(&config, &paths)?;\n let ctx = Self {\n common,\n paths,\n config,\n directories,\n };\n ctx.ensure_directories()?;\n Ok(ctx)\n }\n\n fn init_logging(&self) -> Result<()> {\n if self.common.quiet {\n log::set_max_level(LevelFilter::Off);\n return Ok(());\n }\n\n let mut builder =\n env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(\"info\"));\n\n builder.filter_level(self.effective_log_level());\n\n let force_color = matches!(self.common.color, ColorOption::Always)\n || env::var_os(\"FORCE_COLOR\").is_some();\n let disable_color = self.common.no_color\n || matches!(self.common.color, ColorOption::Never)\n || env::var_os(\"NO_COLOR\").is_some()\n || (!force_color && !io::stderr().is_terminal());\n\n if disable_color {\n builder.write_style(WriteStyle::Never);\n } else if force_color {\n builder.write_style(WriteStyle::Always);\n } else {\n builder.write_style(WriteStyle::Auto);\n }\n\n if self.common.diagnostics {\n builder.format_timestamp_millis();\n builder.format_module_path(true);\n builder.format_target(true);\n }\n\n builder.try_init().or_else(|err| {\n if self.common.verbose > 0 {\n eprintln!(\"logger already initialized: {err}\");\n }\n Ok(())\n })\n }\n\n fn effective_log_level(&self) -> LevelFilter {\n if self.common.trace {\n LevelFilter::Trace\n } else if self.common.debug {\n LevelFilter::Debug\n } else {\n match self.common.verbose {\n 0 => LevelFilter::Info,\n 1 => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n }\n }\n }\n\n fn ensure_directories(&self) -> Result<()> {\n if self.common.dry_run {\n info!(\n \"dry-run: would ensure data dir {}, state dir {}, output dir {}, and index dir {}\",\n self.paths.data_dir.display(),\n self.paths.state_dir.display(),\n self.directories.output_dir.display(),\n self.directories.index_dir.display()\n );\n return Ok(());\n }\n\n fs::create_dir_all(&self.paths.data_dir).with_context(|| {\n format!(\"creating data directory {}\", self.paths.data_dir.display())\n })?;\n fs::create_dir_all(&self.paths.state_dir).with_context(|| {\n format!(\n \"creating state directory {}\",\n self.paths.state_dir.display()\n )\n })?;\n fs::create_dir_all(&self.directories.output_dir).with_context(|| {\n format!(\n \"creating markdown output directory {}\",\n self.directories.output_dir.display()\n )\n })?;\n\n Ok(())\n }\n\n fn pid_path(&self) -> PathBuf {\n self.paths.state_dir.join(\"service.pid\")\n }\n\n fn service_settings(&self, serve: &ServiceRunOpts) -> Result<ServiceSettings> {\n let watch_dir = if let Some(ref provided) = serve.watch_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.watch_dir.clone()\n };\n\n let output_dir = if let Some(ref provided) = serve.output_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.output_dir.clone()\n };\n\n let index_dir = if let Some(ref provided) = serve.index_dir {\n expand_path(provided.clone())?\n } else {\n self.directories.index_dir.clone()\n };\n\n let index_enabled = if serve.disable_index {\n false\n } else {\n self.config.index.enabled\n };\n\n Ok(ServiceSettings {\n watch_dir,\n output_dir,\n index_dir,\n index_enabled,\n debounce: Duration::from_millis(self.config.watcher.debounce_ms.max(50)),\n skip_hidden: self.config.watcher.skip_hidden,\n data_dir: self.paths.data_dir.clone(),\n state_dir: self.paths.state_dir.clone(),\n })\n }\n}\n\n#[derive(Debug, Clone)]\nstruct AppPaths {\n active_config: PathBuf,\n global_config: PathBuf,\n local_config: PathBuf,\n config_override: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl AppPaths {\n fn discover(override_path: Option<PathBuf>) -> Result<Self> {\n let global_config = default_config_dir()?.join(\"config.toml\");\n let local_config = env::current_dir()\n .unwrap_or_else(|_| PathBuf::from(\".\"))\n .join(\"config.toml\");\n\n let config_override = override_path.is_some();\n\n let active_config = match override_path {\n Some(ref path) => expand_path(path.clone())?,\n None => global_config.clone(),\n };\n\n if active_config.parent().is_none() {\n return Err(anyhow!(\"invalid config file path: {active_config:?}\"));\n }\n\n let data_dir = default_data_dir()?;\n let state_dir = default_state_dir()?;\n\n Ok(Self {\n active_config,\n global_config,\n local_config,\n config_override,\n data_dir,\n state_dir,\n })\n }\n\n fn apply_overrides(mut self, cfg: &AppConfig) -> Result<Self> {\n if let Some(ref data_override) = cfg.paths.data_dir {\n self.data_dir = expand_str_path(data_override)?;\n }\n if let Some(ref state_override) = cfg.paths.state_dir {\n self.state_dir = expand_str_path(state_override)?;\n }\n Ok(self)\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct AppConfig {\n profile: String,\n logging: LoggingConfig,\n runtime: RuntimeConfig,\n paths: PathsConfig,\n watcher: WatcherConfig,\n output: OutputConfig,\n index: IndexConfig,\n}\n\nimpl AppConfig {\n fn with_profile_override(mut self, profile: Option<String>) -> Self {\n if let Some(profile) = profile {\n self.profile = profile;\n }\n self\n }\n}\n\nimpl Default for AppConfig {\n fn default() -> Self {\n Self {\n profile: \"default\".to_string(),\n logging: LoggingConfig::default(),\n runtime: RuntimeConfig::default(),\n paths: PathsConfig::default(),\n watcher: WatcherConfig::default(),\n output: OutputConfig::default(),\n index: IndexConfig::default(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct LoggingConfig {\n level: String,\n file: Option<String>,\n}\n\nimpl Default for LoggingConfig {\n fn default() -> Self {\n Self {\n level: \"info\".to_string(),\n file: None,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct RuntimeConfig {\n parallelism: Option<usize>,\n timeout: Option<u64>,\n fail_fast: bool,\n}\n\nimpl Default for RuntimeConfig {\n fn default() -> Self {\n Self {\n parallelism: None,\n timeout: Some(60),\n fail_fast: true,\n }\n }\n}\n\n#[derive(Debug, Default, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct PathsConfig {\n data_dir: Option<String>,\n state_dir: Option<String>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct WatcherConfig {\n watch_dir: String,\n debounce_ms: u64,\n skip_hidden: bool,\n}\n\nimpl Default for WatcherConfig {\n fn default() -> Self {\n Self {\n watch_dir: default_watch_dir_string(),\n debounce_ms: 750,\n skip_hidden: true,\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct OutputConfig {\n markdown_dir: String,\n}\n\nimpl Default for OutputConfig {\n fn default() -> Self {\n Self {\n markdown_dir: default_output_dir_string(),\n }\n }\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(default)]\nstruct IndexConfig {\n enabled: bool,\n index_dir: Option<String>,\n}\n\nimpl Default for IndexConfig {\n fn default() -> Self {\n Self {\n enabled: true,\n index_dir: None,\n }\n }\n}\n\nfn handle_service(ctx: &mut RuntimeContext, command: ServiceCommand) -> Result<()> {\n match command {\n ServiceCommand::Run(opts) => run_service_foreground(ctx, opts),\n ServiceCommand::Start(opts) => start_service_background(ctx, opts),\n ServiceCommand::Stop => stop_service(ctx),\n ServiceCommand::Restart(opts) => {\n stop_service(ctx).ok();\n start_service_background(ctx, opts)\n }\n ServiceCommand::Status => status_service(ctx),\n }\n}\n\nfn run_service_foreground(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let settings = ctx.service_settings(&cmd)?;\n let effective = ctx.config.clone().with_profile_override(None);\n\n info!(\n \"starting {} with profile '{}' watching {} -> {}\",\n APP_NAME,\n effective.profile,\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n\n if settings.index_enabled {\n info!(\"indexing enabled at {}\", settings.index_dir.display());\n } else {\n warn!(\"indexing disabled; searches will not include new documents\");\n }\n\n let mut service = ConversionService::new(settings)?;\n if cmd.once {\n service.process_existing()?;\n service.flush_index()?;\n return Ok(());\n }\n\n service.run()\n}\n\nfn start_service_background(ctx: &mut RuntimeContext, cmd: ServiceRunOpts) -> Result<()> {\n let pid_path = ctx.pid_path();\n if let Some(pid) = read_pid(&pid_path)? {\n if process_running(pid) {\n return Err(anyhow!(\"service already running with pid {}\", pid));\n }\n fs::remove_file(&pid_path).ok();\n }\n\n let settings = ctx.service_settings(&cmd)?;\n if ctx.common.dry_run {\n info!(\n \"dry-run: would start background service watching {} -> {}\",\n settings.watch_dir.display(),\n settings.output_dir.display()\n );\n return Ok(());\n }\n\n let mut command = ProcCommand::new(env::current_exe()?);\n command.arg(\"service\").arg(\"run\");\n append_service_args(&mut command, &cmd);\n if let Some(ref cfg) = ctx.common.config {\n command.arg(\"--config\").arg(cfg);\n }\n\n let log_path = ctx.paths.state_dir.join(\"service.log\");\n let log = OpenOptions::new()\n .create(true)\n .append(true)\n .open(&log_path)\n .with_context(|| format!(\"opening log file {}\", log_path.display()))?;\n let log_err = log.try_clone()?;\n command.stdout(Stdio::from(log));\n command.stderr(Stdio::from(log_err));\n command.stdin(Stdio::null());\n\n let child = command\n .spawn()\n .context(\"spawning background service process\")?;\n write_pid(&pid_path, child.id() as i32)?;\n info!(\n \"started background service pid {} (logs at {})\",\n child.id(),\n log_path.display()\n );\n Ok(())\n}\n\nfn stop_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n info!(\"no running service found (pid file missing)\");\n return Ok(());\n };\n\n if !process_running(pid) {\n info!(\"stale pid {}; removing pid file\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n\n let mut sys = System::new_all();\n sys.refresh_processes();\n let sys_pid = Pid::from_u32(pid as u32);\n if let Some(proc) = sys.process(sys_pid) {\n let killed = proc.kill_with(Signal::Term).unwrap_or(false) || proc.kill();\n if killed {\n info!(\"stopped service pid {}\", pid);\n fs::remove_file(&pid_path).ok();\n return Ok(());\n }\n }\n\n Err(anyhow!(\"failed to stop service pid {}\", pid))\n}\n\nfn status_service(ctx: &RuntimeContext) -> Result<()> {\n let pid_path = ctx.pid_path();\n let Some(pid) = read_pid(&pid_path)? else {\n println!(\"service status: stopped\");\n return Ok(());\n };\n\n if process_running(pid) {\n println!(\"service status: running (pid {})\", pid);\n } else {\n println!(\"service status: not running (stale pid {})\", pid);\n }\n\n Ok(())\n}\n\nfn append_service_args(cmd: &mut ProcCommand, opts: &ServiceRunOpts) {\n if let Some(ref dir) = opts.watch_dir {\n cmd.arg(\"--watch-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.output_dir {\n cmd.arg(\"--output-dir\").arg(dir);\n }\n if let Some(ref dir) = opts.index_dir {\n cmd.arg(\"--index-dir\").arg(dir);\n }\n if opts.disable_index {\n cmd.arg(\"--disable-index\");\n }\n if opts.once {\n cmd.arg(\"--once\");\n }\n}\n\nfn read_pid(path: &Path) -> Result<Option<i32>> {\n if !path.exists() {\n return Ok(None);\n }\n let text = fs::read_to_string(path)?;\n let pid: i32 = text.trim().parse()?;\n Ok(Some(pid))\n}\n\nfn write_pid(path: &Path, pid: i32) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)?;\n }\n fs::write(path, pid.to_string()).with_context(|| format!(\"writing pid file {}\", path.display()))\n}\n\nfn process_running(pid: i32) -> bool {\n let mut sys = System::new();\n sys.refresh_process(Pid::from_u32(pid as u32));\n sys.process(Pid::from_u32(pid as u32)).is_some()\n}\n\nfn handle_search(ctx: &RuntimeContext, cmd: SearchCommand) -> Result<()> {\n let index_dir = if let Some(ref provided) = cmd.index_dir {\n expand_path(provided.clone())?\n } else {\n ctx.directories.index_dir.clone()\n };\n\n let mut index = SearchIndex::open(&index_dir, false)?;\n\n let results = index.search(&cmd.query, cmd.limit)?;\n\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&results).context(\"serializing search results to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&results).context(\"serializing search results to YAML\")?\n );\n } else if results.is_empty() {\n println!(\"No results found\");\n } else {\n for hit in &results {\n println!(\n \"- {} (score {:.2}) -> {}\",\n hit.title\n .as_deref()\n .unwrap_or_else(|| hit.source_path.as_str()),\n hit.score,\n hit.output_path\n );\n }\n }\n\n Ok(())\n}\n\nfn handle_init(ctx: &RuntimeContext, cmd: InitCommand) -> Result<()> {\n if ctx.paths.active_config.exists() && !(cmd.force || ctx.common.assume_yes) {\n return Err(anyhow!(\n \"config already exists at {} (use --force to overwrite)\",\n ctx.paths.active_config.display()\n ));\n }\n\n if ctx.common.dry_run {\n info!(\n \"dry-run: would write default config to {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n\n write_default_config(&ctx.paths.active_config)\n}\n\nfn handle_config(ctx: &RuntimeContext, command: ConfigCommand) -> Result<()> {\n match command {\n ConfigCommand::Show => {\n if ctx.common.json {\n println!(\n \"{}\",\n serde_json::to_string_pretty(&ctx.config)\n .context(\"serializing config to JSON\")?\n );\n } else if ctx.common.yaml {\n println!(\n \"{}\",\n serde_yaml::to_string(&ctx.config).context(\"serializing config to YAML\")?\n );\n } else {\n println!(\"{:#?}\", ctx.config);\n }\n Ok(())\n }\n ConfigCommand::Path => {\n println!(\"{}\", ctx.paths.active_config.display());\n Ok(())\n }\n ConfigCommand::Reset => {\n if ctx.common.dry_run {\n info!(\n \"dry-run: would reset config at {}\",\n ctx.paths.active_config.display()\n );\n return Ok(());\n }\n write_default_config(&ctx.paths.active_config)\n }\n }\n}\n\nfn handle_completions(shell: Shell) -> Result<()> {\n let mut cmd = Cli::command();\n clap_complete::generate(shell, &mut cmd, APP_NAME, &mut io::stdout());\n Ok(())\n}\n\nfn load_or_init_config(paths: &mut AppPaths, common: &CommonOpts) -> Result<AppConfig> {\n if !paths.active_config.exists() {\n if common.dry_run {\n info!(\n \"dry-run: would create default config at {}\",\n paths.active_config.display()\n );\n } else {\n write_default_config(&paths.active_config)?;\n }\n }\n\n let env_prefix = env_prefix();\n let mut builder = Config::builder()\n .set_default(\"profile\", \"default\")?\n .set_default(\"logging.level\", \"info\")?\n .set_default(\"runtime.parallelism\", default_parallelism() as i64)?\n .set_default(\"runtime.timeout\", 60_i64)?\n .set_default(\"runtime.fail_fast\", true)?\n .set_default(\"watcher.watch_dir\", default_watch_dir_string())?\n .set_default(\"watcher.debounce_ms\", 750_i64)?\n .set_default(\"watcher.skip_hidden\", true)?\n .set_default(\"output.markdown_dir\", default_output_dir_string())?\n .set_default(\"index.enabled\", true)?;\n\n if let Some(ref dir) = paths.global_config.parent() {\n fs::create_dir_all(dir)\n .with_context(|| format!(\"creating global config directory {}\", dir.display()))?;\n }\n\n builder = builder.add_source(\n File::from(paths.global_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(\n File::from(paths.local_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n\n builder = builder.add_source(Environment::with_prefix(env_prefix.as_str()).separator(\"__\"));\n\n if paths.config_override {\n builder = builder.add_source(\n File::from(paths.active_config.as_path())\n .format(FileFormat::Toml)\n .required(false),\n );\n }\n\n let built = builder.build()?;\n let mut config: AppConfig = built.try_deserialize()?;\n\n if let Some(ref file) = config.logging.file {\n let expanded = expand_str_path(file)?;\n config.logging.file = Some(expanded.display().to_string());\n }\n\n Ok(config)\n}\n\nfn write_default_config(path: &Path) -> Result<()> {\n if let Some(parent) = path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating config directory {parent:?}\"))?;\n }\n\n let config = AppConfig::default();\n let toml = toml::to_string_pretty(&config).context(\"serializing default config to TOML\")?;\n let mut body = default_config_header(path)?;\n body.push_str(&toml);\n fs::write(path, body).with_context(|| format!(\"writing config file to {}\", path.display()))\n}\n\nfn default_config_header(path: &Path) -> Result<String> {\n let mut buffer = String::new();\n buffer.push_str(\"# Configuration for \");\n buffer.push_str(APP_NAME);\n buffer.push('\\n');\n buffer.push_str(\"# File: \");\n buffer.push_str(&path.display().to_string());\n buffer.push('\\n');\n buffer.push('\\n');\n Ok(buffer)\n}\n\nfn expand_path(path: PathBuf) -> Result<PathBuf> {\n if let Some(text) = path.to_str() {\n expand_str_path(text)\n } else {\n Ok(path)\n }\n}\n\nfn expand_str_path(text: &str) -> Result<PathBuf> {\n let expanded = shellexpand::full(text).context(\"expanding path\")?;\n Ok(PathBuf::from(expanded.to_string()))\n}\n\nfn default_config_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_CONFIG_HOME\").filter(|v| !v.is_empty()) {\n let mut path = PathBuf::from(dir);\n path.push(CONFIG_DIR_NAME);\n return Ok(path);\n }\n\n if let Some(mut dir) = dirs::config_dir() {\n dir.push(CONFIG_DIR_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".config\").join(CONFIG_DIR_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine configuration directory\"))\n}\n\nfn default_data_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_DATA_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::data_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"share\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine data directory\"))\n}\n\nfn default_state_dir() -> Result<PathBuf> {\n if let Some(dir) = env::var_os(\"XDG_STATE_HOME\").filter(|v| !v.is_empty()) {\n return Ok(PathBuf::from(dir).join(APP_NAME));\n }\n\n if let Some(mut dir) = dirs::state_dir() {\n dir.push(APP_NAME);\n return Ok(dir);\n }\n\n dirs::home_dir()\n .map(|home| home.join(\".local\").join(\"state\").join(APP_NAME))\n .ok_or_else(|| anyhow!(\"unable to determine state directory\"))\n}\n\nfn env_prefix() -> String {\n APP_NAME\n .chars()\n .map(|c| {\n if c.is_ascii_alphanumeric() {\n c.to_ascii_uppercase()\n } else {\n '_'\n }\n })\n .collect()\n}\n\nfn default_parallelism() -> usize {\n std::thread::available_parallelism()\n .map(|n| n.get())\n .unwrap_or(1)\n}\n\n#[derive(Debug, Clone)]\nstruct ServiceSettings {\n watch_dir: PathBuf,\n output_dir: PathBuf,\n index_dir: PathBuf,\n index_enabled: bool,\n debounce: Duration,\n skip_hidden: bool,\n data_dir: PathBuf,\n state_dir: PathBuf,\n}\n\nimpl ResolvedDirectories {\n fn from_config(cfg: &AppConfig, paths: &AppPaths) -> Result<Self> {\n let watch_dir = expand_str_path(&cfg.watcher.watch_dir)?;\n let output_dir = expand_str_path(&cfg.output.markdown_dir)?;\n let index_dir = if let Some(ref explicit) = cfg.index.index_dir {\n expand_str_path(explicit)?\n } else {\n paths.data_dir.join(\"index\")\n };\n\n Ok(Self {\n watch_dir,\n output_dir,\n index_dir,\n })\n }\n}\n\nstruct ConversionService {\n settings: ServiceSettings,\n markitdown: MarkItDown,\n indexer: Option<SearchIndex>,\n}\n\nimpl ConversionService {\n fn new(settings: ServiceSettings) -> Result<Self> {\n let indexer = if settings.index_enabled {\n Some(SearchIndex::open(&settings.index_dir, true)?)\n } else {\n None\n };\n\n Ok(Self {\n settings,\n markitdown: MarkItDown::new(),\n indexer,\n })\n }\n\n fn run(&mut self) -> Result<()> {\n self.process_existing()?;\n self.flush_index()?;\n\n let shutdown = Arc::new(AtomicBool::new(false));\n let shutdown_flag = shutdown.clone();\n ctrlc::set_handler(move || {\n shutdown_flag.store(true, Ordering::SeqCst);\n })\n .context(\"installing ctrl+c handler\")?;\n\n let (tx, rx) = mpsc::channel();\n let mut watcher = self.build_watcher(tx)?;\n\n watcher\n .watch(self.settings.watch_dir.as_path(), RecursiveMode::Recursive)\n .with_context(|| format!(\"watching directory {}\", self.settings.watch_dir.display()))?;\n\n info!(\"watching for changes (press Ctrl+C to stop)\");\n while !shutdown.load(Ordering::SeqCst) {\n match rx.recv_timeout(self.settings.debounce) {\n Ok(Ok(event)) => {\n self.handle_event(event)?;\n self.flush_index()?;\n }\n Ok(Err(err)) => warn!(\"watch error: {err}\"),\n Err(RecvTimeoutError::Timeout) => continue,\n Err(RecvTimeoutError::Disconnected) => break,\n }\n }\n\n self.flush_index()\n }\n\n fn build_watcher(\n &self,\n tx: mpsc::Sender<Result<Event, notify::Error>>,\n ) -> Result<RecommendedWatcher> {\n notify::recommended_watcher(move |res| {\n let _ = tx.send(res);\n })\n .context(\"creating filesystem watcher\")\n }\n\n fn process_existing(&mut self) -> Result<()> {\n for entry in WalkDir::new(&self.settings.watch_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n if self.should_skip(path) {\n continue;\n }\n self.process_path(path)?;\n }\n\n // Also index any existing markdown outputs (so prior conversions are searchable\n // even if the source files haven't changed).\n if self.settings.index_enabled {\n self.index_existing_outputs()?;\n }\n Ok(())\n }\n\n fn handle_event(&mut self, event: Event) -> Result<()> {\n let kind = event.kind.clone();\n for path in event.paths {\n if !path.is_file() {\n continue;\n }\n if self.should_skip(&path) {\n continue;\n }\n match kind {\n EventKind::Create(_) | EventKind::Modify(_) => {\n self.process_path(&path)?;\n }\n _ => {}\n }\n }\n Ok(())\n }\n\n fn should_skip(&self, path: &Path) -> bool {\n if self.settings.watch_dir != self.settings.output_dir\n && path.starts_with(&self.settings.output_dir)\n {\n return true;\n }\n\n if !self.settings.skip_hidden {\n return false;\n }\n\n path.components().any(|component| {\n component\n .as_os_str()\n .to_str()\n .map(|s| s.starts_with('.'))\n .unwrap_or(false)\n })\n }\n\n fn process_path(&mut self, path: &Path) -> Result<()> {\n let output_path = self.output_path_for(path)?;\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n\n if is_markdown && output_path == path {\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n if !self.needs_processing(path, &output_path)? {\n debug!(\"skipping {} (already up to date)\", path.display());\n if let Some(indexer) = self.indexer.as_mut() {\n Self::index_existing_output(indexer, &output_path, path)?;\n }\n return Ok(());\n }\n\n let converted = match self.markitdown.convert(\n path.to_str()\n .ok_or_else(|| anyhow!(\"invalid path encoding for {}\", path.display()))?,\n None,\n ) {\n Some(result) => result,\n None => {\n warn!(\"no converter available for {}\", path.display());\n return Ok(());\n }\n };\n\n let converted_at = OffsetDateTime::now_utc()\n .format(&Rfc3339)\n .unwrap_or_else(|_| \"unknown\".to_string());\n\n let source_modified = fs::metadata(path)\n .ok()\n .and_then(|meta| meta.modified().ok())\n .and_then(system_time_to_rfc3339);\n\n let frontmatter = Frontmatter {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n source_modified,\n title: converted.title.clone(),\n converted_at: converted_at.clone(),\n };\n\n let yaml = serde_yaml::to_string(&frontmatter).context(\"serializing frontmatter\")?;\n let mut body = String::new();\n body.push_str(\"---\\n\");\n body.push_str(&yaml);\n body.push_str(\"---\\n\\n\");\n body.push_str(&converted.text_content);\n\n if self.settings.data_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps data directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if self.settings.state_dir == self.settings.output_dir {\n warn!(\n \"output directory {} overlaps state directory; consider separating them\",\n self.settings.output_dir.display()\n );\n }\n\n if let Some(parent) = output_path.parent() {\n fs::create_dir_all(parent)\n .with_context(|| format!(\"creating output directory {}\", parent.display()))?;\n }\n\n fs::write(&output_path, body)\n .with_context(|| format!(\"writing markdown to {}\", output_path.display()))?;\n info!(\"converted {} -> {}\", path.display(), output_path.display());\n\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.index_document(&IndexedDocument {\n source_path: path.display().to_string(),\n output_path: output_path.display().to_string(),\n title: converted.title.clone(),\n content: converted.text_content.clone(),\n converted_at: Some(converted_at.clone()),\n })?;\n }\n\n Ok(())\n }\n\n fn needs_processing(&self, source: &Path, output: &Path) -> Result<bool> {\n if !output.exists() {\n return Ok(true);\n }\n\n let source_meta = fs::metadata(source)?;\n let output_meta = fs::metadata(output)?;\n let source_time = source_meta.modified().ok();\n let output_time = output_meta.modified().ok();\n\n Ok(match (source_time, output_time) {\n (Some(src), Some(out)) => src > out,\n _ => true,\n })\n }\n\n fn output_path_for(&self, source: &Path) -> Result<PathBuf> {\n let relative = source\n .strip_prefix(&self.settings.watch_dir)\n .unwrap_or(source);\n let mut output = self.settings.output_dir.join(relative);\n output.set_extension(\"md\");\n Ok(output)\n }\n\n fn flush_index(&mut self) -> Result<()> {\n if let Some(indexer) = self.indexer.as_mut() {\n indexer.commit()?;\n }\n Ok(())\n }\n\n fn index_existing_output(\n indexer: &mut SearchIndex,\n output_path: &Path,\n source_path: &Path,\n ) -> Result<()> {\n let body = match fs::read_to_string(output_path) {\n Ok(b) => b,\n Err(err) => {\n warn!(\n \"unable to read {} for indexing: {err}\",\n output_path.display()\n );\n return Ok(());\n }\n };\n\n let (frontmatter, content) = parse_frontmatter(&body);\n let source_path_str = frontmatter\n .as_ref()\n .map(|fm| fm.source_path.clone())\n .unwrap_or_else(|| source_path.display().to_string());\n\n let indexed = IndexedDocument {\n source_path: source_path_str,\n output_path: output_path.display().to_string(),\n title: frontmatter.as_ref().and_then(|fm| fm.title.clone()),\n content: content.to_string(),\n converted_at: frontmatter.as_ref().map(|fm| fm.converted_at.clone()),\n };\n\n indexer.index_document(&indexed)\n }\n\n fn index_existing_outputs(&mut self) -> Result<()> {\n let Some(indexer) = self.indexer.as_mut() else {\n return Ok(());\n };\n\n for entry in WalkDir::new(&self.settings.output_dir)\n .into_iter()\n .filter_map(Result::ok)\n {\n let path = entry.path();\n if !path.is_file() {\n continue;\n }\n let is_markdown = path\n .extension()\n .and_then(|e| e.to_str())\n .map(|ext| ext.eq_ignore_ascii_case(\"md\"))\n .unwrap_or(false);\n if !is_markdown {\n continue;\n }\n let source_path = path; // will be overwritten by frontmatter if present\n let _ = Self::index_existing_output(indexer, path, source_path);\n }\n Ok(())\n }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\nstruct Frontmatter {\n source_path: String,\n output_path: String,\n source_modified: Option<String>,\n title: Option<String>,\n converted_at: String,\n}\n\nfn system_time_to_rfc3339(time: SystemTime) -> Option<String> {\n let datetime: OffsetDateTime = time.into();\n datetime.format(&Rfc3339).ok()\n}\n\nfn parse_frontmatter(body: &str) -> (Option<Frontmatter>, &str) {\n if let Some(rest) = body.strip_prefix(\"---\\n\") {\n if let Some(idx) = rest.find(\"\\n---\\n\") {\n let (front, content) = rest.split_at(idx);\n let content = &content[\"\\n---\\n\".len()..];\n if let Ok(parsed) = serde_yaml::from_str::<Frontmatter>(front) {\n return (Some(parsed), content);\n }\n }\n }\n (None, body)\n}\n\nfn default_watch_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"Documents\").display().to_string())\n .unwrap_or_else(|| \"~/Documents\".to_string())\n}\n\nfn default_output_dir_string() -> String {\n dirs::home_dir()\n .map(|home| home.join(\"markdown\").display().to_string())\n .unwrap_or_else(|| \"~/markdown\".to_string())\n}\n\nimpl fmt::Display for AppPaths {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(\n f,\n \"config: {}, data: {}, state: {}\",\n self.active_config.display(),\n self.data_dir.display(),\n self.state_dir.display()\n )\n }\n}\n"}}, "ingestr-cli/examples/config.toml": {"b580df1d8ac145689a4d7496bf05d90608329fb0": {"t": 1785746801.0200865, "n": 1081, "h": "f1b57caabf42d9aa", "c": "# Example configuration for the ingestr markdown service.\n# Copy to $XDG_CONFIG_HOME/ingestr-cli/config.toml (or ~/.config/ingestr-cli/config.toml).\n\nprofile = \"default\"\n\n[logging]\n# Valid levels: error, warn, info, debug, trace\nlevel = \"info\"\n# file = \"~/Library/Logs/ingestr-cli.log\"\n\n[runtime]\n# parallelism = 8\ntimeout = 60\nfail_fast = true\n\n[watcher]\nwatch_dir = \"~/Documents\"\ndebounce_ms = 750\nskip_hidden = true\n\n[output]\nmarkdown_dir = \"~/markdown\"\n\n[index]\nenabled = true\n# index_dir = \"$XDG_DATA_HOME/ingestr-cli/index\"\n\n[paths]\n# data_dir = \"$XDG_DATA_HOME/ingestr-cli\"\n# state_dir = \"$XDG_STATE_HOME/ingestr-cli\"\n\n[llm]\n# Enable LLM-based image description generation\nenabled = false\n# LLM provider: \"openai\", \"gemini\", or \"deepseek\"\nclient = \"openai\"\n# Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\nmodel = \"gpt-4o\"\n# Custom API base URL (for OpenAI-compatible endpoints, local LLMs, etc.)\n# base_url = \"http://localhost:11434/v1\"\n# API key (can also use environment variables: OPENAI_API_KEY, GEMINI_API_KEY, DEEPSEEK_API_KEY)\n# api_key = \"sk-...\"\n"}, "f08c782bfa3b8770235e6b7cc65cd8251ca1a70a": {"t": 1785757400.0059433, "n": 1081, "h": "f1b57caabf42d9aa", "c": "# Example configuration for the ingestr markdown service.\n# Copy to $XDG_CONFIG_HOME/ingestr-cli/config.toml (or ~/.config/ingestr-cli/config.toml).\n\nprofile = \"default\"\n\n[logging]\n# Valid levels: error, warn, info, debug, trace\nlevel = \"info\"\n# file = \"~/Library/Logs/ingestr-cli.log\"\n\n[runtime]\n# parallelism = 8\ntimeout = 60\nfail_fast = true\n\n[watcher]\nwatch_dir = \"~/Documents\"\ndebounce_ms = 750\nskip_hidden = true\n\n[output]\nmarkdown_dir = \"~/markdown\"\n\n[index]\nenabled = true\n# index_dir = \"$XDG_DATA_HOME/ingestr-cli/index\"\n\n[paths]\n# data_dir = \"$XDG_DATA_HOME/ingestr-cli\"\n# state_dir = \"$XDG_STATE_HOME/ingestr-cli\"\n\n[llm]\n# Enable LLM-based image description generation\nenabled = false\n# LLM provider: \"openai\", \"gemini\", or \"deepseek\"\nclient = \"openai\"\n# Model name (e.g., \"gpt-4o\", \"gemini-1.5-flash\", \"deepseek-chat\")\nmodel = \"gpt-4o\"\n# Custom API base URL (for OpenAI-compatible endpoints, local LLMs, etc.)\n# base_url = \"http://localhost:11434/v1\"\n# API key (can also use environment variables: OPENAI_API_KEY, GEMINI_API_KEY, DEEPSEEK_API_KEY)\n# api_key = \"sk-...\"\n"}, "983c06cc50811c79a4fd476475acbf0808a10bdc": {"t": 1785757401.3336155, "n": 625, "h": "a9923e73b479954e", "c": "# Example configuration for the ingestr markdown service.\n# Copy to $XDG_CONFIG_HOME/ingestr-cli/config.toml (or ~/.config/ingestr-cli/config.toml).\n\nprofile = \"default\"\n\n[logging]\n# Valid levels: error, warn, info, debug, trace\nlevel = \"info\"\n# file = \"~/Library/Logs/ingestr-cli.log\"\n\n[runtime]\n# parallelism = 8\ntimeout = 60\nfail_fast = true\n\n[watcher]\nwatch_dir = \"~/Documents\"\ndebounce_ms = 750\nskip_hidden = true\n\n[output]\nmarkdown_dir = \"~/markdown\"\n\n[index]\nenabled = true\n# index_dir = \"$XDG_DATA_HOME/ingestr-cli/index\"\n\n[paths]\n# data_dir = \"$XDG_DATA_HOME/ingestr-cli\"\n# state_dir = \"$XDG_STATE_HOME/ingestr-cli\"\n"}}}} |