alexion
e35ed9d05b
Introduce the thinnest complete path that benchmarks a skill's efficacy against a no-skill baseline and renders an HTML report. - tests/ convention: a top-level tree mirroring skills/ by full path, with case directories holding a case.md (CASE-FORMAT.md) and an optional fixture. - benchmark-skill: a distributed, non-model-invocable runner (/benchmark-skill <name>) that orchestrates force-invoked new-skill and skill-absent baseline arms plus a blind per-trial judge as in-session subagents. - Deterministic core (Python): a pure transform over the collected run data that sums per-arm usage, collapses each trial to WIN/TIE/LOSS, applies the efficacy pass rule (wins >= 3, losses <= 1), and renders a self-contained report. Parameterized over arms and comparisons; two-arm here. - Fixture-driven no-LLM check (checks/benchmark-core.nix) wired into flake.nix. - Real tests tree for axi-review; a live run produced its efficacy report.
98 lines
3.8 KiB
Nix
98 lines
3.8 KiB
Nix
{
|
|
description = "Personal agent skills, packaged through Nix.";
|
|
|
|
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
|
inputs.flake-utils.url = "github:numtide/flake-utils";
|
|
|
|
# Present only so `nix flake check` can evaluate `homeModules.default` against
|
|
# real home-manager (the home-manager-module check). Its nixpkgs follows this
|
|
# flake's, so the module is checked against the same nixpkgs-and-home-manager
|
|
# pairing a consumer following this flake would get. It has no bearing on the
|
|
# module a consumer imports — that module is a bare function taking the
|
|
# importing configuration's own home-manager and pkgs.
|
|
inputs.home-manager.url = "github:nix-community/home-manager";
|
|
inputs.home-manager.inputs.nixpkgs.follows = "nixpkgs";
|
|
|
|
outputs =
|
|
{ self, nixpkgs, flake-utils, home-manager }:
|
|
let
|
|
mkSkill = import ./lib/mk-skill.nix;
|
|
mkSkillsShellHook = import ./lib/mk-skills-shell-hook.nix;
|
|
discoverSkills = import ./lib/discover.nix nixpkgs.lib;
|
|
|
|
discoveredSkills = discoverSkills ./skills;
|
|
|
|
skillPackages =
|
|
pkgs:
|
|
nixpkgs.lib.listToAttrs (
|
|
map (skill: {
|
|
inherit (skill) name;
|
|
value = mkSkill { inherit pkgs; inherit (skill) name src; };
|
|
}) discoveredSkills
|
|
);
|
|
in
|
|
{
|
|
lib = { inherit mkSkill mkSkillsShellHook; };
|
|
|
|
# Global placement: an operator selects skills into `~/.claude/skills/`
|
|
# via `programs.agents.skills`. Not per-system — it is a module function,
|
|
# and the skills it places come from the importing configuration's own
|
|
# selection rather than this flake's nixpkgs.
|
|
homeModules = rec {
|
|
agents-skills = ./home-manager-module.nix;
|
|
default = agents-skills;
|
|
};
|
|
}
|
|
//
|
|
# An explicit system list rather than `eachDefaultSystem`.
|
|
# That default set includes x86_64-darwin, which current nixpkgs no longer
|
|
# evaluates (its `legacyPackages` throws).
|
|
flake-utils.lib.eachSystem
|
|
[
|
|
"x86_64-linux"
|
|
"aarch64-linux"
|
|
"aarch64-darwin"
|
|
]
|
|
(
|
|
system:
|
|
let
|
|
pkgs = nixpkgs.legacyPackages.${system};
|
|
in
|
|
{
|
|
# No `default`: a catalog has no single default skill.
|
|
packages = skillPackages pkgs;
|
|
|
|
# Each discovered skill is folded in as its own check, so
|
|
# `nix flake check` builds every skill and fails on a malformed one.
|
|
checks = (skillPackages pkgs) // {
|
|
skill-build = import ./checks/skill-build.nix {
|
|
inherit pkgs mkSkill discoverSkills;
|
|
};
|
|
|
|
# Proves the global-placement module composes: it builds the
|
|
# home-files derivation `homeModules.default` produces under several
|
|
# operator configurations. Reuses this flake's `mkSkill` so the
|
|
# sample skills carry the same name contract real ones do.
|
|
home-manager-module = import ./checks/home-manager-module.nix {
|
|
inherit pkgs home-manager mkSkill;
|
|
module = self.homeModules.default;
|
|
};
|
|
|
|
# Proves the per-project dev-shell helper: it runs the produced hook
|
|
# against a fixture project and asserts placement, stateless
|
|
# stale-removal, and that hand-authored skills stay untouched.
|
|
shell-hook = import ./checks/shell-hook.nix {
|
|
inherit pkgs mkSkill mkSkillsShellHook;
|
|
};
|
|
|
|
# Proves the benchmark skill's deterministic core at its single seam:
|
|
# it feeds committed fixtures to the core and asserts the metrics, the
|
|
# Efficacy verdict, and the rendered report.
|
|
benchmark-core = import ./checks/benchmark-core.nix {
|
|
inherit pkgs;
|
|
};
|
|
};
|
|
}
|
|
);
|
|
}
|