diff --git a/docs/03 wrk/ariadne.md b/docs/03 wrk/ariadne.md index 227ad18..f04ea2e 100644 --- a/docs/03 wrk/ariadne.md +++ b/docs/03 wrk/ariadne.md @@ -249,43 +249,45 @@ Why discard/rework it: - Bazel runfiles handling made the shell script messy; - the PR should be refocused before this work continues. +## Clew artifact boundary + +Clew now owns extraction, preparation, and Ren'Py AST export. +Ariadne owns analysis over Clew-produced data. + +Ariadne consumes Clew's raw `clew-renpy-ast-v1` JSON directly. +It deserializes Clew's generic serializer format into typed Rust structs and enums. +It then maps the full Clew AST into Ariadne-owned internal data structures. + +The first importer preserves unrecognized or unneeded Ren'Py objects. +Use an `UnknownNode` or equivalent representation instead of dropping data. +Semantic helpers can exist for known Ren'Py statement classes. + +`NodeKey.namespace` comes from the Clew AST file path relative to `output/ast/`. +Clew preserves the game-root boundary in that path. +This keeps label names from different source universes separate. + +The first Ariadne CLI prints a human-readable validation report by default. +The report includes counts for documents, namespaces, and nodes by kind. +It also includes counts for unknown nodes, edges, refs, and warnings. +The CLI also supports an opt-in debug dump of internal state, for example `--dump-state`. +Treat that dump as a development surface until Ariadne documents a stable format. + +Route planning is out of scope for the first importer slice. + ## Preparation output direction -Preparation output should evolve from: +Clew produces the preparation output for Ariadne. +The current relevant output is: ```text output/ - source/ - work/ - logs/ + ast/ # Clew `clew-renpy-ast-v1` JSON + extracted/ # extracted game files ``` -to: - -```text -output/ - ast/ # normalized Ren'Py AST JSON - source/ # decompiled/debug source - work/ # extracted raw/intermediate files - logs/ -``` - -Decompiled source remains useful as a debug artifact, but may stop being the primary parser input. - -## Preparation tooling direction - -The shell script is becoming too broad. A better next step is to move preparation into its own Python tool before returning to Ariadne core. - -That preparation tool should own: - -- game-root and launcher discovery; -- archive extraction; -- `.rpyc` collection; -- AST export through the matching bundled Ren'Py runtime; -- optional decompilation for debug source; -- output layout and logs. - -Ariadne core should only depend on the preparation tool's normalized artifacts. +Decompiled source is no longer Ariadne's primary input. +The old `src/ariadne/extract-and-decompile.sh` script was obsolete. +Ariadne now relies on Clew for extraction and preparation. ## Python dependencies and vendoring @@ -306,7 +308,8 @@ The AST exporter itself needs only the Python standard library plus the target l ## Recommended next session start 1. Do not continue from the discarded PR implementation. -2. Inspect the actual `rpycdec` source and decide how to expose it under `//third_party/python` or another `//third_party` package. -3. Design a standalone preparation Python tool with no unnecessary external dependencies. -4. Keep the normalized AST JSON boundary between bundled Ren'Py runtime code and Rust Ariadne core. -5. Return to the Rust `State` shape after the preparation artifact contract is clearer. +2. Implement typed Rust deserialization for Clew's `clew-renpy-ast-v1` JSON. +3. Map every Clew AST value into Ariadne-owned internal data structures. +4. Preserve unknown Ren'Py objects as `UnknownNode` or an equivalent node. +5. Build the default validation report for imported Clew output. +6. Add an opt-in debug dump for internal state. diff --git a/src/ariadne/BUILD b/src/ariadne/BUILD index c3f343f..276d40d 100644 --- a/src/ariadne/BUILD +++ b/src/ariadne/BUILD @@ -13,8 +13,3 @@ rust_binary( deps = [":ariadne_lib"], ) -sh_binary( - name = "extract_and_decompile", - srcs = ["extract-and-decompile.sh"], - data = ["extract-tools.nix"], -) diff --git a/src/ariadne/README.md b/src/ariadne/README.md index bbadaaa..efc85c6 100644 --- a/src/ariadne/README.md +++ b/src/ariadne/README.md @@ -1,27 +1,18 @@ # Ariadne -Ariadne prepares Ren'Py game source for later analysis. +Ariadne analyzes Clew output for a local Ren'Py game install. + +Keep project-owned source under `src/ariadne/`. +Keep game-derived files local and ignored under `src/ariadne/output/`. + +Clew owns extraction, preparation, and Ren'Py AST export. +Ariadne consumes Clew's `clew-renpy-ast-v1` JSON from `output/ast/`. +Ariadne maps that data into internal analysis state. ## Bazel shape -Keep project-owned source under `src/ariadne/`. Keep game-derived files local and ignored under -`src/ariadne/output/`. - -The first target is a shell entry point: +Run the current placeholder CLI with: ```sh -bazel run //src/ariadne:extract_and_decompile -- \ - --game-root /path/to/game-install +bazel run //src/ariadne:ariadne ``` - -The script defaults to `/src/ariadne/output`. -This path stays stable when you run the command from inside `src/ariadne`. -Use the game installation directory as the game root. -One run processes all installed season directories. -Extracted archive source keeps the archive path relative to the game root, so files with the same -internal path from different season installs do not overwrite each other. -If `rpycdec` is not available, the script uses `extract-tools.nix` to enter a Nix shell. - -Keep the script as a thin orchestration layer. -Bazel owns stable inputs and wrappers. -Keep large game files and recovered output outside the build graph. diff --git a/src/ariadne/extract-and-decompile.sh b/src/ariadne/extract-and-decompile.sh deleted file mode 100755 index 8aca368..0000000 --- a/src/ariadne/extract-and-decompile.sh +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'USAGE' -Usage: extract-and-decompile --game-root [--output-root ] [--force] - -Prepare local game-owned source for Ariadne. - -Use the game installation directory as the game root to process all installed seasons. -The utility writes recovered source under /source/, transient work under /work/, and local logs under /logs/. -The default output root is /src/ariadne/output. -Without --force, it refuses to run when any of those directories already exists. -With --force, it deletes and recreates only source/, work/, and logs/ under the output root. -The script enters a Nix shell for rpycdec when required tools are not already available. -Game-derived preparation output must remain local and must not be committed. -USAGE -} - -fail() { - printf 'extract-and-decompile: %s\n' "$*" >&2 - exit 2 -} - -require_tool() { - local name=$1 - if ! command -v "$name" >/dev/null 2>&1; then - fail "missing required tool '$name'" - fi -} - -tools_available() { - command -v python3 >/dev/null 2>&1 && command -v rpycdec >/dev/null 2>&1 -} - -enter_nix_shell_if_needed() { - if tools_available; then - return - fi - - if [[ "${ARIADNE_NIX_SHELL:-}" == 1 ]]; then - require_tool python3 - require_tool rpycdec - return - fi - - require_tool nix - - local tools_nix - if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then - tools_nix=$BUILD_WORKSPACE_DIRECTORY/src/ariadne/extract-tools.nix - else - tools_nix=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/extract-tools.nix - fi - - [[ -f "$tools_nix" ]] || fail "missing Ariadne tool definition: $tools_nix" - - say "entering Nix shell for rpycdec" - export ARIADNE_NIX_SHELL=1 - exec nix shell \ - --extra-experimental-features 'nix-command flakes' \ - --file "$tools_nix" \ - --command bash "$0" "$@" -} - -say() { - printf 'extract-and-decompile: %s\n' "$*" >&2 -} - -game_root= -script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then - default_output_root=$BUILD_WORKSPACE_DIRECTORY/src/ariadne/output -else - default_output_root=$script_dir/output -fi -output_root=$default_output_root -force=false -original_args=("$@") - -while [[ $# -gt 0 ]]; do - case "$1" in - --game-root) - [[ $# -ge 2 ]] || fail '--game-root requires a path' - game_root=$2 - shift 2 - ;; - --output-root) - [[ $# -ge 2 ]] || fail '--output-root requires a path' - output_root=$2 - shift 2 - ;; - --force) - force=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - fail "unexpected argument: $1" - ;; - esac -done - -[[ -n "$game_root" ]] || fail 'missing --game-root ' -[[ -d "$game_root" ]] || fail "game root is not a directory: $game_root" - -enter_nix_shell_if_needed "${original_args[@]}" - -preparation_dirs=(source work logs) - -for name in "${preparation_dirs[@]}"; do - path="$output_root/$name" - if [[ -e "$path" && "$force" != true ]]; then - fail "refusing to overwrite existing preparation path without --force: $path" - fi -done - -if [[ "$force" == true ]]; then - say "removing existing preparation directories under $output_root" - for name in "${preparation_dirs[@]}"; do - rm -rf -- "${output_root:?}/$name" - done -fi - -say "creating preparation directories under $output_root" -for name in "${preparation_dirs[@]}"; do - mkdir -p -- "$output_root/$name" -done - -source_dir=$output_root/source -work_dir=$output_root/work -logs_dir=$output_root/logs -summary_log=$logs_dir/extract-and-decompile.log -: > "$summary_log" - -log() { - printf '%s\n' "$*" >> "$summary_log" -} - -record() { - say "$*" - log "$*" -} - -relative_to_game_root() { - realpath --relative-to="$game_root" "$1" -} - -is_renpy_runtime_path() { - local relative=$1 - [[ "$relative" == renpy/* || "$relative" == */renpy/* ]] -} - -source_output_path() { - local relative=$1 - local destination_prefix=${2:-} - if [[ -n "$destination_prefix" ]]; then - printf '%s/%s/%s\n' "$source_dir" "$destination_prefix" "$relative" - else - printf '%s/%s\n' "$source_dir" "$relative" - fi -} - -source_log_relative() { - local relative=$1 - local destination_prefix=${2:-} - if [[ -n "$destination_prefix" ]]; then - printf '%s/%s\n' "$destination_prefix" "$relative" - else - printf '%s\n' "$relative" - fi -} - -copy_source_file() { - local input=$1 - local input_root=$2 - local label=$3 - local destination_prefix=${4:-} - local relative output log_relative - relative=$(realpath --relative-to="$input_root" "$input") - log_relative=$(source_log_relative "$relative" "$destination_prefix") - if is_renpy_runtime_path "$relative"; then - log "skipped $label Ren'Py runtime source: $log_relative" - return - fi - output=$(source_output_path "$relative" "$destination_prefix") - mkdir -p -- "$(dirname -- "$output")" - cp -- "$input" "$output" - record "copied $label source: $log_relative" -} - -copy_loose_source() { - copy_source_file "$1" "$game_root" loose -} - -decompile_compiled_script() { - local input=$1 - local input_root=$2 - local destination_prefix=${3:-} - local relative output_dir log_file log_relative - relative=$(realpath --relative-to="$input_root" "$input") - log_relative=$(source_log_relative "$relative" "$destination_prefix") - if is_renpy_runtime_path "$relative"; then - log "skipped Ren'Py runtime compiled script: $log_relative" - return - fi - output_dir=$(source_output_path "$(dirname -- "$relative")" "$destination_prefix") - log_file=$logs_dir/decompile-${log_relative//\//__}.log - mkdir -p -- "$output_dir" - say "decompiling script: $log_relative" - RPYCDEC_NO_WARNING=1 rpycdec decompile --output "$output_dir" "$input" > "$log_file" 2>&1 - log "decompiled script: $log_relative" -} - -extract_archive() { - local archive=$1 - local relative archive_output log_file destination_prefix - relative=$(relative_to_game_root "$archive") - archive_output=$work_dir/archives/${relative%.rpa} - log_file=$logs_dir/unrpa-${relative//\//__}.log - destination_prefix=$(dirname -- "$relative") - if [[ "$destination_prefix" == "." ]]; then - destination_prefix= - fi - mkdir -p -- "$archive_output" - say "extracting archive: $relative" - RPYCDEC_NO_WARNING=1 rpycdec unrpa --output "$archive_output" "$archive" > "$log_file" 2>&1 - log "extracted archive: $relative" - - while IFS= read -r -d '' archived_source; do - copy_source_file "$archived_source" "$archive_output" archived "$destination_prefix" - done < <(find "$archive_output" -type f -name '*.rpy' -print0 | sort -z) - - while IFS= read -r -d '' compiled; do - decompile_compiled_script "$compiled" "$archive_output" "$destination_prefix" - done < <(find "$archive_output" -type f \( -name '*.rpyc' -o -name '*.rpymc' \) -print0 | sort -z) -} - -say "scanning loose source files" -while IFS= read -r -d '' source_file; do - copy_loose_source "$source_file" -done < <(find "$game_root" -type f -name '*.rpy' -print0 | sort -z) - -say "scanning archives" -while IFS= read -r -d '' archive; do - extract_archive "$archive" -done < <(find "$game_root" -type f -name '*.rpa' -print0 | sort -z) - -say "scanning loose compiled scripts" -while IFS= read -r -d '' compiled; do - decompile_compiled_script "$compiled" "$game_root" -done < <(find "$game_root" -type f \( -name '*.rpyc' -o -name '*.rpymc' \) -print0 | sort -z) - -say "done" diff --git a/src/ariadne/extract-tools.nix b/src/ariadne/extract-tools.nix deleted file mode 100644 index 2f20db7..0000000 --- a/src/ariadne/extract-tools.nix +++ /dev/null @@ -1,29 +0,0 @@ -let - pkgs = import (builtins.getFlake "nixpkgs") { system = builtins.currentSystem; }; - rpycdec = pkgs.python3Packages.buildPythonApplication { - pname = "rpycdec"; - version = "0.0.0+1000bd1"; - pyproject = true; - src = pkgs.fetchFromGitHub { - owner = "cnfatal"; - repo = "rpycdec"; - rev = "1000bd1da10ef07e82f6fa560c621e6a4b55d1b8"; - hash = "sha256-VoVbIKs8xz5x2s0OGqdHbY55fWDruqMAX8YYjEQhwTQ="; - }; - nativeBuildInputs = with pkgs.python3Packages; [ - hatch-vcs - hatchling - ]; - env.SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RPYCDEC = "0.0.0+1000bd1"; - }; -in -pkgs.buildEnv { - name = "ariadne-extract-and-decompile-tools"; - paths = [ - pkgs.bash - pkgs.coreutils - pkgs.findutils - pkgs.python3 - rpycdec - ]; -}