diff --git a/AGENTS.md b/AGENTS.md index aab6be4..9deeba5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,3 +120,7 @@ Do not use `git subtree add` or `git subtree pull` for Pi PR branches. Gitea rebases the synthetic upstream root commit without the `third_party/pi` prefix. Preserve `git-subtree-dir` and `git-subtree-split` trailers on Pi vendor commits. +- Xatu file discovery skips symlinks, including Bazel runfiles. + Copy Nix fixtures into `TEST_TMPDIR` before passing their paths to CLI tests. +- The vendored `rustversion-1.0.23/build/` files are currently ignored and untracked. + Fresh worktrees cannot build Xatu tests until those upstream files are restored. diff --git a/Cargo.lock b/Cargo.lock index bedb295..377e144 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -402,6 +402,8 @@ dependencies = [ "monorepo", "rnix", "rowan", + "serde", + "serde_json", "status", ] diff --git a/src/xatu/BUILD b/src/xatu/BUILD index f60733b..79bd95c 100644 --- a/src/xatu/BUILD +++ b/src/xatu/BUILD @@ -16,6 +16,7 @@ rust_library( deps = [ "//src/status:status", "//third_party/crates:rnix", + "//third_party/crates:serde", "//third_party/crates/rowan-0.16.1:rowan", ], ) @@ -41,6 +42,23 @@ rust_test( ], ) +rust_test( + name = "debug_test", + srcs = ["debug_test.rs"], + data = [ + ":xatu", + "testdata/clean.json", + "testdata/clean.nix", + "testdata/invalid.nix", + "testdata/lint_findings.json", + "testdata/lint_findings.nix", + ], + deps = [ + "//third_party/crates:googletest", + "//third_party/crates:serde_json", + ], +) + rust_binary( name = "xatu", crate_name = "xatu_cli", @@ -50,5 +68,6 @@ rust_binary( "//src/monorepo:monorepo", "//src/status:status", "//third_party/crates:clap", + "//third_party/crates:serde_json", ], ) diff --git a/src/xatu/Cargo.toml b/src/xatu/Cargo.toml index d12e186..81e2b62 100644 --- a/src/xatu/Cargo.toml +++ b/src/xatu/Cargo.toml @@ -16,6 +16,8 @@ monorepo = { workspace = true } rnix = { workspace = true } rowan = { workspace = true } status = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } [dev-dependencies] googletest = { workspace = true } diff --git a/src/xatu/README.md b/src/xatu/README.md new file mode 100644 index 0000000..bbef586 --- /dev/null +++ b/src/xatu/README.md @@ -0,0 +1,40 @@ +# Xatu debug output + +Run `bazel run //src/xatu:xatu -- --debug src/dotfiles/registry.nix` to inspect parser output. +The command writes JSON containing file paths and complete `ParsedFile` values: + +- File IDs. +- Lowered rule nodes and tokens. +- Parse errors. + +It skips lint rules and leaves the normal analysis report unchanged. +It sorts files by path and removes duplicate paths before assigning file IDs. + +Parse errors remain in JSON and produce exit code 2. +Read errors go to stderr and produce exit code 2 without JSON output. +Successful parsing produces exit code 0, including files with potential lint findings. +The debug format is an inspection interface, not a stable interchange schema. + +## Tests + +Run `bazel test //src/xatu:all`. +Existing black-box tests check lint findings. +`debug_test.rs` invokes the CLI and compares parsed JSON against expected fixture output. +The test also checks parse errors, read errors, and deterministic file ordering. +It copies Nix fixtures into `TEST_TMPDIR` because discovery skips symlinks, including Bazel runfiles. + +Regenerate expected JSON after intentional parser or fixture changes: + +```bash +bazel build //src/xatu:xatu +binary="$(pwd)/bazel-bin/src/xatu/xatu" +( + cd src/xatu + for fixture in clean lint_findings; do + "$binary" --debug "testdata/$fixture.nix" > "testdata/$fixture.json" + done +) +``` + +Review regenerated output before accepting it. +Expected JSON files are generated test artifacts; do not edit them manually. diff --git a/src/xatu/ast.rs b/src/xatu/ast.rs index 1deb464..787be4a 100644 --- a/src/xatu/ast.rs +++ b/src/xatu/ast.rs @@ -1,6 +1,8 @@ +use serde::Serialize; + use crate::FileId; -#[derive(Debug)] +#[derive(Debug, Serialize)] pub struct ParsedFile { pub file_id: FileId, pub nodes: Vec, @@ -8,13 +10,13 @@ pub struct ParsedFile { pub errors: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Node { pub range: SourceRange, pub data: NodeData, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] pub struct SourceRange { pub start: usize, pub end: usize, @@ -26,46 +28,46 @@ impl std::fmt::Display for SourceRange { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum NodeData { Binding(Binding), } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Binding { pub key: Ident, pub value: Expr, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Ident { pub name: String, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum Expr { Ident(Ident), Select(Select), } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Select { pub receiver: String, pub attr: Ident, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Token { pub range: SourceRange, pub data: TokenData, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum TokenData { UnquotedUri(String), } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ParseError { pub range: SourceRange, pub message: String, diff --git a/src/xatu/debug_test.rs b/src/xatu/debug_test.rs new file mode 100644 index 0000000..25b10b7 --- /dev/null +++ b/src/xatu/debug_test.rs @@ -0,0 +1,89 @@ +use std::path::PathBuf; +use std::process::{Command, Output}; + +use googletest::matchers::eq; +use googletest::{OrFail, expect_that, gtest}; +use serde_json::Value; + +#[gtest] +fn debug_matches_fixture_representations() -> googletest::Result<()> { + for name in ["clean", "lint_findings"] { + let output = run_debug(&[&format!("testdata/{name}.nix")])?; + expect_that!(output.status.code(), eq(Some(0))); + expect_that!(output.stderr.is_empty(), eq(true)); + let actual: Value = serde_json::from_slice(&output.stdout).or_fail()?; + let expected: Value = serde_json::from_str( + &std::fs::read_to_string(fixture_dir()?.join(format!("testdata/{name}.json"))) + .or_fail()?, + ) + .or_fail()?; + expect_that!(actual, eq(&expected)); + } + Ok(()) +} + +#[gtest] +fn debug_retains_parse_errors() -> googletest::Result<()> { + let output = run_debug(&["testdata/invalid.nix"])?; + expect_that!(output.status.code(), eq(Some(2))); + expect_that!(output.stderr.is_empty(), eq(true)); + let actual: Value = serde_json::from_slice(&output.stdout).or_fail()?; + expect_that!( + actual["files"][0]["parsed"]["errors"] + .as_array() + .or_fail()? + .is_empty(), + eq(false) + ); + Ok(()) +} + +#[gtest] +fn debug_orders_and_deduplicates_files() -> googletest::Result<()> { + let first = run_debug(&[ + "testdata/lint_findings.nix", + "testdata/clean.nix", + "testdata/clean.nix", + ])?; + let second = run_debug(&["testdata/clean.nix", "testdata/lint_findings.nix"])?; + expect_that!(first.status.code(), eq(Some(0))); + expect_that!(second.status.code(), eq(Some(0))); + expect_that!(first.stdout, eq(&second.stdout)); + Ok(()) +} + +#[gtest] +fn debug_reports_io_errors_on_stderr() -> googletest::Result<()> { + let output = run_debug(&["testdata/missing.nix"])?; + expect_that!(output.status.code(), eq(Some(2))); + expect_that!(output.stdout.is_empty(), eq(true)); + expect_that!(output.stderr.is_empty(), eq(false)); + Ok(()) +} + +fn run_debug(paths: &[&str]) -> googletest::Result { + let directory = fixture_dir()?; + static NEXT_ID: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let id = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let working = PathBuf::from(std::env::var("TEST_TMPDIR").or_fail()?).join(id.to_string()); + std::fs::create_dir_all(working.join("testdata")).or_fail()?; + // Discovery skips symlinks, including Bazel's fixture runfiles. + for name in ["clean", "lint_findings", "invalid"] { + let path = format!("testdata/{name}.nix"); + std::fs::copy(directory.join(&path), working.join(&path)).or_fail()?; + } + let output = Command::new(directory.join("xatu")) + .current_dir(&working) + .env_remove("BUILD_WORKSPACE_DIRECTORY") + .arg("--debug") + .args(paths) + .output(); + std::fs::remove_dir_all(working).or_fail()?; + output.or_fail() +} + +fn fixture_dir() -> googletest::Result { + Ok(PathBuf::from(std::env::var("TEST_SRCDIR").or_fail()?) + .join(std::env::var("TEST_WORKSPACE").or_fail()?) + .join("src/xatu")) +} diff --git a/src/xatu/file_store.rs b/src/xatu/file_store.rs index ef2aafe..c701afa 100644 --- a/src/xatu/file_store.rs +++ b/src/xatu/file_store.rs @@ -29,5 +29,5 @@ pub struct SourceFile { pub source: String, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] pub struct FileId(usize); diff --git a/src/xatu/testdata/clean.json b/src/xatu/testdata/clean.json new file mode 100644 index 0000000..5d043e8 --- /dev/null +++ b/src/xatu/testdata/clean.json @@ -0,0 +1,32 @@ +{ + "files": [ + { + "path": "testdata/clean.nix", + "parsed": { + "file_id": 0, + "nodes": [ + { + "range": { + "start": 85, + "end": 103 + }, + "data": { + "Binding": { + "key": { + "name": "different" + }, + "value": { + "Ident": { + "name": "other" + } + } + } + } + } + ], + "tokens": [], + "errors": [] + } + } + ] +} diff --git a/src/xatu/testdata/lint_findings.json b/src/xatu/testdata/lint_findings.json new file mode 100644 index 0000000..12d3646 --- /dev/null +++ b/src/xatu/testdata/lint_findings.json @@ -0,0 +1,63 @@ +{ + "files": [ + { + "path": "testdata/lint_findings.nix", + "parsed": { + "file_id": 0, + "nodes": [ + { + "range": { + "start": 4, + "end": 22 + }, + "data": { + "Binding": { + "key": { + "name": "package" + }, + "value": { + "Ident": { + "name": "package" + } + } + } + } + }, + { + "range": { + "start": 25, + "end": 42 + }, + "data": { + "Binding": { + "key": { + "name": "from" + }, + "value": { + "Select": { + "receiver": "pkgs", + "attr": { + "name": "from" + } + } + } + } + } + } + ], + "tokens": [ + { + "range": { + "start": 51, + "end": 70 + }, + "data": { + "UnquotedUri": "https://example.com" + } + } + ], + "errors": [] + } + } + ] +} diff --git a/src/xatu/xatu.rs b/src/xatu/xatu.rs index bc8b40f..cc3ee5d 100644 --- a/src/xatu/xatu.rs +++ b/src/xatu/xatu.rs @@ -29,6 +29,43 @@ impl std::fmt::Display for Report { } } +/// Contains parser output without lint diagnostics. +#[derive(Debug, serde::Serialize)] +pub struct DebugReport { + files: Vec, +} + +impl DebugReport { + pub fn has_parse_errors(&self) -> bool { + self.files.iter().any(|file| !file.parsed.errors.is_empty()) + } +} + +#[derive(Debug, serde::Serialize)] +struct DebugFile { + path: std::path::PathBuf, + parsed: ast::ParsedFile, +} + +/// Collects parser output without running lint rules. +pub fn debug(mut files: Vec) -> DebugReport { + files.sort_by(|left, right| left.path.cmp(&right.path)); + files.dedup_by(|left, right| left.path == right.path); + let mut store = FileStore::default(); + let files = files + .into_iter() + .map(|file| { + let file_id = store.add(file); + let file = store.get(file_id); + DebugFile { + path: file.path.clone(), + parsed: parser::parse_file(file_id, &file.source), + } + }) + .collect(); + DebugReport { files } +} + /// Analyzes source files and returns the diagnostics Xatu finds. pub fn future_sight(files: Vec) -> status::Result { let mut store = FileStore::default(); diff --git a/src/xatu/xatu_main.rs b/src/xatu/xatu_main.rs index 256984d..2383f04 100644 --- a/src/xatu/xatu_main.rs +++ b/src/xatu/xatu_main.rs @@ -2,12 +2,36 @@ use clap::Parser; use status::ResultExt; use std::path::{Path, PathBuf}; use std::process::ExitCode; -use xatu::{Report, SourceFile}; +use xatu::SourceFile; fn main() -> ExitCode { let args = Args::parse(); - match analyze(args) { + let debug = args.debug; + let files = match collect_files(args) { + Ok(files) => files, + Err(error) => { + eprintln!("{error}"); + return ExitCode::from(2); + } + }; + if debug { + let report = xatu::debug(files); + match serde_json::to_string_pretty(&report) { + Ok(json) => println!("{json}"), + Err(error) => { + eprintln!("{error}"); + return ExitCode::from(2); + } + } + return if report.has_parse_errors() { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + }; + } + + match xatu::future_sight(files) { Ok(report) if report.is_clean() => { println!("No findings."); ExitCode::SUCCESS @@ -25,11 +49,14 @@ fn main() -> ExitCode { #[derive(Debug, Parser)] struct Args { + /// Dumps parser representations as JSON instead of running lint rules. + #[arg(long)] + debug: bool, #[arg(value_name = "PATH", default_value = ".")] paths: Vec, } -fn analyze(args: Args) -> status::Result { +fn collect_files(args: Args) -> status::Result> { let paths: Vec = args .paths .into_iter() @@ -44,7 +71,7 @@ fn analyze(args: Args) -> status::Result { .flatten() .collect(); - xatu::future_sight(files) + Ok(files) } fn collect_source_files(path: &Path) -> status::Result> {