feat(ariadne): export Ren'Py AST artifacts

This commit is contained in:
2026-09-06 09:14:24 -04:00
parent 6b408cd66d
commit d5a5121355
7 changed files with 784 additions and 4 deletions

View File

@@ -7,11 +7,11 @@ Usage: extract-and-decompile --game-root <path> [--output-root <path>] [--force]
Prepare local game-owned source for Ariadne. Prepare local game-owned source for Ariadne.
Use the game installation directory as the game root to process all installed seasons. Use the game installation directory as the game root to process all installed launchers.
The utility writes recovered source under <output-root>/source/, transient work under <output-root>/work/, and local logs under <output-root>/logs/. The utility writes recovered source under <output-root>/source/, normalized Ren'Py AST exports under <output-root>/ast/, transient work under <output-root>/work/, and local logs under <output-root>/logs/.
The default output root is <workspace>/src/ariadne/output. The default output root is <workspace>/src/ariadne/output.
Without --force, it refuses to run when any of those directories already exists. Without --force, it refuses to run when any of those directories already exists.
With --force, it deletes and recreates only source/, work/, and logs/ under the output root. With --force, it deletes and recreates only source/, ast/, work/, and logs/ under the output root.
The script enters a Nix shell for rpycdec when required tools are not already available. The script enters a Nix shell for rpycdec when required tools are not already available.
Game-derived preparation output must remain local and must not be committed. Game-derived preparation output must remain local and must not be committed.
USAGE USAGE
@@ -109,7 +109,7 @@ done
enter_nix_shell_if_needed "${original_args[@]}" enter_nix_shell_if_needed "${original_args[@]}"
preparation_dirs=(source work logs) preparation_dirs=(source ast work logs)
for name in "${preparation_dirs[@]}"; do for name in "${preparation_dirs[@]}"; do
path="$output_root/$name" path="$output_root/$name"
@@ -131,6 +131,7 @@ for name in "${preparation_dirs[@]}"; do
done done
source_dir=$output_root/source source_dir=$output_root/source
ast_dir=$output_root/ast
work_dir=$output_root/work work_dir=$output_root/work
logs_dir=$output_root/logs logs_dir=$output_root/logs
summary_log=$logs_dir/extract-and-decompile.log summary_log=$logs_dir/extract-and-decompile.log
@@ -174,6 +175,43 @@ source_log_relative() {
fi fi
} }
launcher_prefix() {
local log_relative=$1
printf '%s\n' "${log_relative%%/*}"
}
launcher_root_for() {
local log_relative=$1
local prefix
prefix=$(launcher_prefix "$log_relative")
printf '%s/%s\n' "$game_root" "$prefix"
}
bundled_python_for() {
local launcher_root=$1
local candidate
for candidate in \
"$launcher_root/lib/linux-x86_64/python" \
"$launcher_root/lib/linux-i686/python"; do
if [[ -x "$candidate" ]]; then
printf '%s\n' "$candidate"
return
fi
done
fail "no bundled Linux Python found under launcher root: $launcher_root"
}
run_bundled_python() {
local python=$1
shift
python=$(realpath "$python")
if command -v steam-run >/dev/null 2>&1; then
steam-run "$python" "$@"
else
"$python" "$@"
fi
}
copy_source_file() { copy_source_file() {
local input=$1 local input=$1
local input_root=$2 local input_root=$2
@@ -196,6 +234,28 @@ copy_loose_source() {
copy_source_file "$1" "$game_root" loose copy_source_file "$1" "$game_root" loose
} }
export_ast() {
local input=$1
local log_relative=$2
local launcher_root python adapter output log_file namespace
launcher_root=$(launcher_root_for "$log_relative")
python=$(bundled_python_for "$launcher_root")
adapter=$script_dir/tools/export-renpy-ast.py
namespace=$(launcher_prefix "$log_relative")
output=$ast_dir/${log_relative%.*}.json
log_file=$logs_dir/ast-${log_relative//\//__}.log
[[ -f "$adapter" ]] || fail "missing Ren'Py AST adapter: $adapter"
mkdir -p -- "$(dirname -- "$output")"
say "exporting Ren'Py AST: $log_relative"
run_bundled_python "$python" "$adapter" \
--renpy-root "$launcher_root" \
--namespace "$namespace" \
--rpyc "$input" \
--slot 1 \
--output "$output" > "$log_file" 2>&1
log "exported Ren'Py AST: $log_relative"
}
decompile_compiled_script() { decompile_compiled_script() {
local input=$1 local input=$1
local input_root=$2 local input_root=$2
@@ -207,6 +267,7 @@ decompile_compiled_script() {
log "skipped Ren'Py runtime compiled script: $log_relative" log "skipped Ren'Py runtime compiled script: $log_relative"
return return
fi fi
export_ast "$input" "$log_relative"
output_dir=$(source_output_path "$(dirname -- "$relative")" "$destination_prefix") output_dir=$(source_output_path "$(dirname -- "$relative")" "$destination_prefix")
log_file=$logs_dir/decompile-${log_relative//\//__}.log log_file=$logs_dir/decompile-${log_relative//\//__}.log
mkdir -p -- "$output_dir" mkdir -p -- "$output_dir"

View File

@@ -0,0 +1,114 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "anyhow"
version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "memchr"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "proc-macro2"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
[[package]]
name = "renpy-ast-adapter-prototype"
version = "0.1.0"
dependencies = [
"anyhow",
"serde",
"serde_json",
]
[[package]]
name = "serde"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "syn"
version = "3.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

View File

@@ -0,0 +1,11 @@
[package]
name = "renpy-ast-adapter-prototype"
version = "0.1.0"
edition = "2024"
[workspace]
[dependencies]
anyhow = "1.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

View File

@@ -0,0 +1,191 @@
#!/usr/bin/env python
"""Export a Ren'Py AST pickle to Ariadne-neutral JSON records.
Run this with the Python interpreter bundled with the target Ren'Py launcher.
The script intentionally emits a small normalized record set instead of exposing
Ren'Py objects to Rust.
"""
from __future__ import print_function
import argparse
import json
import os
import pickle
import struct
import sys
import zlib
from collections import Counter
try:
text_type = unicode # noqa: F821 - Python 2 runtime.
except NameError:
text_type = str
RPYC2_HEADER = b"RENPY RPC2"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--renpy-root", required=True, help="Launcher root containing the renpy package")
parser.add_argument("--namespace", required=True, help="Namespace assigned to exported nodes")
parser.add_argument("--rpyc", required=True, help="Compiled Ren'Py script file")
parser.add_argument("--slot", type=int, default=1, choices=(1, 2), help="RPYC slot to export")
parser.add_argument("--output", required=True, help="JSON output path")
return parser.parse_args()
def install_minimal_renpy_runtime(renpy):
class Obj(object):
pass
script = Obj()
script.all_pyexpr = []
script.all_pycode = []
script.record_pycode = False
log = Obj()
log.mutated = {}
game = Obj()
game.script = script
game.log = log
renpy.game = game
def read_rpyc_slot(path, slot):
with open(path, "rb") as handle:
data = handle.read()
if not data.startswith(RPYC2_HEADER):
if slot != 1:
raise ValueError("legacy rpyc files only have slot 1")
return zlib.decompress(data)
pos = len(RPYC2_HEADER)
while True:
header_slot, start, length = struct.unpack("III", data[pos : pos + 12])
if header_slot == slot:
return zlib.decompress(data[start : start + length])
if header_slot == 0:
raise ValueError("slot {0} not found".format(slot))
pos += 12
def py_string(value):
if value is None:
return None
if isinstance(value, bytes):
return value.decode("utf-8", "replace")
return text_type(value)
def simple_value(value):
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, (str, bytes)):
return py_string(value)
if isinstance(value, tuple):
return [simple_value(item) for item in value]
if isinstance(value, list):
return [simple_value(item) for item in value]
return py_string(value)
def node_fields(node):
kind = type(node).__name__
if kind == "Label":
return {"name": py_string(node.name)}
if kind == "Say":
return {
"who": py_string(node.who),
"what": py_string(node.what),
"identifier": py_string(getattr(node, "identifier", None)),
}
if kind == "Jump":
return {"target": py_string(node.target), "expression": bool(node.expression)}
if kind == "Call":
return {"label": py_string(node.label), "expression": bool(node.expression)}
if kind == "If":
return {"conditions": [simple_value(condition) for condition, _block in node.entries]}
if kind == "Menu":
return {
"items": [
{"label": py_string(label), "condition": simple_value(condition), "has_block": block is not None}
for label, condition, block in node.items
]
}
if kind == "Python":
return {"source": py_string(node.code.source), "store": py_string(node.store), "hide": bool(node.hide)}
return {}
def flatten(stmts):
nodes = []
for stmt in stmts:
stmt.get_children(nodes.append)
return nodes
def main():
args = parse_args()
renpy_root = os.path.abspath(args.renpy_root)
sys.path.insert(0, renpy_root)
import renpy # pylint: disable=import-error,import-outside-toplevel
import renpy.object # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.ast # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.python # pylint: disable=import-error,unused-import,import-outside-toplevel
install_minimal_renpy_runtime(renpy)
payload = pickle.loads(read_rpyc_slot(args.rpyc, args.slot))
metadata, stmts = payload
nodes = flatten(stmts)
line_counts = Counter((py_string(node.filename), int(node.linenumber)) for node in nodes)
seen = Counter()
records = []
for node in nodes:
path = py_string(node.filename)
line = int(node.linenumber)
occurrence = seen[(path, line)]
seen[(path, line)] += 1
records.append(
{
"key": {
"namespace": args.namespace,
"path": path,
"line": line,
"occurrence": occurrence,
},
"kind": type(node).__name__,
"fields": node_fields(node),
}
)
export = {
"schema": "ariadne-renpy-ast-adapter-prototype-v1",
"namespace": args.namespace,
"renpy_version": py_string(renpy.version),
"script_version": metadata.get("version"),
"slot": args.slot,
"source": os.path.abspath(args.rpyc),
"node_count": len(records),
"ambiguous_source_locations": sum(1 for count in line_counts.values() if count > 1),
"records": records,
}
output = os.path.abspath(args.output)
parent = os.path.dirname(output)
if parent and not os.path.isdir(parent):
os.makedirs(parent)
with open(output, "w") as handle:
json.dump(export, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps({key: export[key] for key in ("renpy_version", "script_version", "node_count", "ambiguous_source_locations")}, indent=2, sort_keys=True))
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,49 @@
# Ren'Py AST adapter prototype
This throwaway prototype tests option 2 from the Ariadne design discussion: use the target launcher's own Python and Ren'Py modules to load Ren'Py AST pickle data, export normalized JSON records, and let Rust import those records into Ariadne-shaped state.
The prototype keeps Ren'Py internals outside Rust. Rust consumes only adapter JSON.
## Shape tested
```text
.rpyc slot
-> bundled Python adapter
-> normalized AST JSON records
-> Rust State { nodes, edges, pending }
```
The Rust state mirrors the current design sketch:
```rust
State {
nodes: Map<NodeKey, Node>,
edges: Map<NodeKey, Set<NodeKey>>,
pending: Vec<Operation>,
}
```
## Run
Run the adapter with the Python interpreter bundled with the target launcher. On NixOS, wrap that interpreter with `steam-run`.
```sh
steam-run /path/to/launcher/lib/linux-x86_64/python \
prototypes/renpy-ast-adapter/export_renpy_ast.py \
--renpy-root /path/to/launcher \
--namespace example \
--rpyc /path/to/script.rpyc \
--slot 1 \
--output /tmp/ariadne-renpy-ast-export.json
cargo run --manifest-path prototypes/renpy-ast-adapter/Cargo.toml -- \
/tmp/ariadne-renpy-ast-export.json
```
Use local ignored paths for target-game files. Do not commit generated exports.
## Early observation
Ren'Py AST can contain multiple nodes with the same source file and line. Examples include transition/display nodes sharing a physical source line, and slot 2 translation wrappers around `Say` nodes.
The prototype therefore adds a per-source-location `occurrence` field to `NodeKey`. This is evidence for revisiting whether Ariadne's final source key needs a similar disambiguator when AST ingestion is used.

View File

@@ -0,0 +1,163 @@
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::fs;
use anyhow::{Context, Result};
use serde::Deserialize;
use serde_json::Value;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize)]
struct NodeKey {
namespace: String,
path: String,
line: u32,
occurrence: u32,
}
#[derive(Debug, Clone)]
struct Node {
kind: String,
fields: Value,
}
#[derive(Debug)]
struct State {
nodes: BTreeMap<NodeKey, Node>,
edges: BTreeMap<NodeKey, BTreeSet<NodeKey>>,
pending: Vec<Operation>,
}
#[derive(Debug)]
enum Operation {
AddNode { key: NodeKey, node: Node },
}
#[derive(Debug, Deserialize)]
struct ExportFile {
schema: String,
namespace: String,
renpy_version: String,
script_version: u64,
slot: u8,
source: String,
node_count: usize,
ambiguous_source_locations: usize,
records: Vec<Record>,
}
#[derive(Debug, Deserialize)]
struct Record {
key: NodeKey,
kind: String,
fields: Value,
}
impl State {
fn empty() -> Self {
Self {
nodes: BTreeMap::new(),
edges: BTreeMap::new(),
pending: Vec::new(),
}
}
fn queue(&mut self, operation: Operation) {
self.pending.push(operation);
}
fn apply_pending(&mut self) {
for operation in self.pending.drain(..) {
match operation {
Operation::AddNode { key, node } => {
self.nodes.insert(key, node);
}
}
}
}
}
fn import_export(state: &mut State, export: ExportFile) {
for record in export.records {
state.queue(Operation::AddNode {
key: record.key,
node: Node {
kind: record.kind,
fields: record.fields,
},
});
}
}
fn main() -> Result<()> {
let path = env::args()
.nth(1)
.context("usage: renpy-ast-adapter-prototype <renpy-ast-export.json>")?;
let text = fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
let export: ExportFile = serde_json::from_str(&text).with_context(|| format!("parse {path}"))?;
println!("schema: {}", export.schema);
println!("namespace: {}", export.namespace);
println!("renpy version: {}", export.renpy_version);
println!("script version: {}", export.script_version);
println!("slot: {}", export.slot);
println!("source: {}", export.source);
println!("exported nodes: {}", export.node_count);
println!(
"source locations with multiple AST nodes: {}",
export.ambiguous_source_locations
);
let mut state = State::empty();
import_export(&mut state, export);
println!("pending operations before apply: {}", state.pending.len());
state.apply_pending();
println!("state nodes after apply: {}", state.nodes.len());
println!("state edge sources after apply: {}", state.edges.len());
let mut by_kind: BTreeMap<&str, usize> = BTreeMap::new();
for node in state.nodes.values() {
*by_kind.entry(node.kind.as_str()).or_default() += 1;
}
println!("nodes by kind:");
for (kind, count) in by_kind {
println!(" {kind}: {count}");
}
println!("first nodes:");
for (key, node) in state.nodes.iter().take(20) {
println!(
" {}:{}#{} {} {}",
key.path,
key.line,
key.occurrence,
node.kind,
compact_fields(&node.fields)
);
}
Ok(())
}
fn compact_fields(fields: &Value) -> String {
let Some(object) = fields.as_object() else {
return String::new();
};
let mut parts = Vec::new();
for (key, value) in object.iter().take(3) {
if value.is_null() {
continue;
}
parts.push(format!("{key}={}", short_json(value)));
}
parts.join(" ")
}
fn short_json(value: &Value) -> String {
let mut text = value.to_string();
const LIMIT: usize = 80;
if text.len() > LIMIT {
text.truncate(LIMIT);
text.push('…');
}
text
}

View File

@@ -0,0 +1,191 @@
#!/usr/bin/env python
"""Export a Ren'Py AST pickle to Ariadne-neutral JSON records.
Run this with the Python interpreter bundled with the target Ren'Py launcher.
The script intentionally emits a small normalized record set instead of exposing
Ren'Py objects to Rust.
"""
from __future__ import print_function
import argparse
import json
import os
import pickle
import struct
import sys
import zlib
from collections import Counter
try:
text_type = unicode # noqa: F821 - Python 2 runtime.
except NameError:
text_type = str
RPYC2_HEADER = b"RENPY RPC2"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--renpy-root", required=True, help="Launcher root containing the renpy package")
parser.add_argument("--namespace", required=True, help="Namespace assigned to exported nodes")
parser.add_argument("--rpyc", required=True, help="Compiled Ren'Py script file")
parser.add_argument("--slot", type=int, default=1, choices=(1, 2), help="RPYC slot to export")
parser.add_argument("--output", required=True, help="JSON output path")
return parser.parse_args()
def install_minimal_renpy_runtime(renpy):
class Obj(object):
pass
script = Obj()
script.all_pyexpr = []
script.all_pycode = []
script.record_pycode = False
log = Obj()
log.mutated = {}
game = Obj()
game.script = script
game.log = log
renpy.game = game
def read_rpyc_slot(path, slot):
with open(path, "rb") as handle:
data = handle.read()
if not data.startswith(RPYC2_HEADER):
if slot != 1:
raise ValueError("legacy rpyc files only have slot 1")
return zlib.decompress(data)
pos = len(RPYC2_HEADER)
while True:
header_slot, start, length = struct.unpack("III", data[pos : pos + 12])
if header_slot == slot:
return zlib.decompress(data[start : start + length])
if header_slot == 0:
raise ValueError("slot {0} not found".format(slot))
pos += 12
def py_string(value):
if value is None:
return None
if isinstance(value, bytes):
return value.decode("utf-8", "replace")
return text_type(value)
def simple_value(value):
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, (str, bytes)):
return py_string(value)
if isinstance(value, tuple):
return [simple_value(item) for item in value]
if isinstance(value, list):
return [simple_value(item) for item in value]
return py_string(value)
def node_fields(node):
kind = type(node).__name__
if kind == "Label":
return {"name": py_string(node.name)}
if kind == "Say":
return {
"who": py_string(node.who),
"what": py_string(node.what),
"identifier": py_string(getattr(node, "identifier", None)),
}
if kind == "Jump":
return {"target": py_string(node.target), "expression": bool(node.expression)}
if kind == "Call":
return {"label": py_string(node.label), "expression": bool(node.expression)}
if kind == "If":
return {"conditions": [simple_value(condition) for condition, _block in node.entries]}
if kind == "Menu":
return {
"items": [
{"label": py_string(label), "condition": simple_value(condition), "has_block": block is not None}
for label, condition, block in node.items
]
}
if kind == "Python":
return {"source": py_string(node.code.source), "store": py_string(node.store), "hide": bool(node.hide)}
return {}
def flatten(stmts):
nodes = []
for stmt in stmts:
stmt.get_children(nodes.append)
return nodes
def main():
args = parse_args()
renpy_root = os.path.abspath(args.renpy_root)
sys.path.insert(0, renpy_root)
import renpy # pylint: disable=import-error,import-outside-toplevel
import renpy.object # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.ast # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.python # pylint: disable=import-error,unused-import,import-outside-toplevel
install_minimal_renpy_runtime(renpy)
payload = pickle.loads(read_rpyc_slot(args.rpyc, args.slot))
metadata, stmts = payload
nodes = flatten(stmts)
line_counts = Counter((py_string(node.filename), int(node.linenumber)) for node in nodes)
seen = Counter()
records = []
for node in nodes:
path = py_string(node.filename)
line = int(node.linenumber)
occurrence = seen[(path, line)]
seen[(path, line)] += 1
records.append(
{
"key": {
"namespace": args.namespace,
"path": path,
"line": line,
"occurrence": occurrence,
},
"kind": type(node).__name__,
"fields": node_fields(node),
}
)
export = {
"schema": "ariadne-renpy-ast-adapter-prototype-v1",
"namespace": args.namespace,
"renpy_version": py_string(renpy.version),
"script_version": metadata.get("version"),
"slot": args.slot,
"source": os.path.abspath(args.rpyc),
"node_count": len(records),
"ambiguous_source_locations": sum(1 for count in line_counts.values() if count > 1),
"records": records,
}
output = os.path.abspath(args.output)
parent = os.path.dirname(output)
if parent and not os.path.isdir(parent):
os.makedirs(parent)
with open(output, "w") as handle:
json.dump(export, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps({key: export[key] for key in ("renpy_version", "script_version", "node_count", "ambiguous_source_locations")}, indent=2, sort_keys=True))
if __name__ == "__main__":
main()