alexion/dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ariadne): export Ren'Py AST artifacts

Browse Source
This commit is contained in:
alexion
2026-09-06 09:14:24 -04:00
parent 6b408cd66d
commit d5a5121355
7 changed files with 784 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/ariadne/extract-and-decompile.sh
View File

@@ -7,11 +7,11 @@ Usage: extract-and-decompile --game-root <path> [--output-root <path>] [--force]
Prepare local game-owned source for Ariadne.
Use the game installation directory as the game root to process all installed seasons.
The utility writes recovered source under <output-root>/source/, transient work under <output-root>/work/, and local logs under <output-root>/logs/.
Use the game installation directory as the game root to process all installed launchers.
The utility writes recovered source under <output-root>/source/, normalized Ren'Py AST exports under <output-root>/ast/, transient work under <output-root>/work/, and local logs under <output-root>/logs/.
The default output root is <workspace>/src/ariadne/output.
Without --force, it refuses to run when any of those directories already exists.
With --force, it deletes and recreates only source/, work/, and logs/ under the output root.
With --force, it deletes and recreates only source/, ast/, work/, and logs/ under the output root.
The script enters a Nix shell for rpycdec when required tools are not already available.
Game-derived preparation output must remain local and must not be committed.
USAGE
@@ -109,7 +109,7 @@ done
enter_nix_shell_if_needed "${original_args[@]}"
preparation_dirs=(source work logs)
preparation_dirs=(source ast work logs)
for name in "${preparation_dirs[@]}"; do
path="$output_root/$name"
@@ -131,6 +131,7 @@ for name in "${preparation_dirs[@]}"; do
done
source_dir=$output_root/source
ast_dir=$output_root/ast
work_dir=$output_root/work
logs_dir=$output_root/logs
summary_log=$logs_dir/extract-and-decompile.log
@@ -174,6 +175,43 @@ source_log_relative() {
fi
}
launcher_prefix() {
local log_relative=$1
printf '%s\n' "${log_relative%%/*}"
}
launcher_root_for() {
local log_relative=$1
local prefix
prefix=$(launcher_prefix "$log_relative")
printf '%s/%s\n' "$game_root" "$prefix"
}
bundled_python_for() {
local launcher_root=$1
local candidate
for candidate in \
"$launcher_root/lib/linux-x86_64/python" \
"$launcher_root/lib/linux-i686/python"; do
if [[ -x "$candidate" ]]; then
printf '%s\n' "$candidate"
return
fi
done
fail "no bundled Linux Python found under launcher root: $launcher_root"
}
run_bundled_python() {
local python=$1
shift
python=$(realpath "$python")
if command -v steam-run >/dev/null 2>&1; then
steam-run "$python" "$@"
else
"$python" "$@"
fi
}
copy_source_file() {
local input=$1
local input_root=$2
@@ -196,6 +234,28 @@ copy_loose_source() {
copy_source_file "$1" "$game_root" loose
}
export_ast() {
local input=$1
local log_relative=$2
local launcher_root python adapter output log_file namespace
launcher_root=$(launcher_root_for "$log_relative")
python=$(bundled_python_for "$launcher_root")
adapter=$script_dir/tools/export-renpy-ast.py
namespace=$(launcher_prefix "$log_relative")
output=$ast_dir/${log_relative%.*}.json
log_file=$logs_dir/ast-${log_relative//\//__}.log
[[ -f "$adapter" ]] || fail "missing Ren'Py AST adapter: $adapter"
mkdir -p -- "$(dirname -- "$output")"
say "exporting Ren'Py AST: $log_relative"
run_bundled_python "$python" "$adapter" \
--renpy-root "$launcher_root" \
--namespace "$namespace" \
--rpyc "$input" \
--slot 1 \
--output "$output" > "$log_file" 2>&1
log "exported Ren'Py AST: $log_relative"
}
decompile_compiled_script() {
local input=$1
local input_root=$2
@@ -207,6 +267,7 @@ decompile_compiled_script() {
log "skipped Ren'Py runtime compiled script: $log_relative"
return
fi
export_ast "$input" "$log_relative"
output_dir=$(source_output_path "$(dirname -- "$relative")" "$destination_prefix")
log_file=$logs_dir/decompile-${log_relative//\//__}.log
mkdir -p -- "$output_dir"

114
src/ariadne/prototypes/renpy-ast-adapter/Cargo.lock generated Normal file
View File

@@ -0,0 +1,114 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "anyhow"
version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "memchr"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "proc-macro2"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
[[package]]
name = "renpy-ast-adapter-prototype"
version = "0.1.0"
dependencies = [
"anyhow",
"serde",
"serde_json",
]
[[package]]
name = "serde"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "syn"
version = "3.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

11
src/ariadne/prototypes/renpy-ast-adapter/Cargo.toml Normal file
View File

@@ -0,0 +1,11 @@
[package]
name = "renpy-ast-adapter-prototype"
version = "0.1.0"
edition = "2024"
[workspace]
[dependencies]
anyhow = "1.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

191
src/ariadne/prototypes/renpy-ast-adapter/export_renpy_ast.py Normal file
View File

@@ -0,0 +1,191 @@
#!/usr/bin/env python
"""Export a Ren'Py AST pickle to Ariadne-neutral JSON records.
Run this with the Python interpreter bundled with the target Ren'Py launcher.
The script intentionally emits a small normalized record set instead of exposing
Ren'Py objects to Rust.
"""
from __future__ import print_function
import argparse
import json
import os
import pickle
import struct
import sys
import zlib
from collections import Counter
try:
text_type = unicode # noqa: F821 - Python 2 runtime.
except NameError:
text_type = str
RPYC2_HEADER = b"RENPY RPC2"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--renpy-root", required=True, help="Launcher root containing the renpy package")
parser.add_argument("--namespace", required=True, help="Namespace assigned to exported nodes")
parser.add_argument("--rpyc", required=True, help="Compiled Ren'Py script file")
parser.add_argument("--slot", type=int, default=1, choices=(1, 2), help="RPYC slot to export")
parser.add_argument("--output", required=True, help="JSON output path")
return parser.parse_args()
def install_minimal_renpy_runtime(renpy):
class Obj(object):
pass
script = Obj()
script.all_pyexpr = []
script.all_pycode = []
script.record_pycode = False
log = Obj()
log.mutated = {}
game = Obj()
game.script = script
game.log = log
renpy.game = game
def read_rpyc_slot(path, slot):
with open(path, "rb") as handle:
data = handle.read()
if not data.startswith(RPYC2_HEADER):
if slot != 1:
raise ValueError("legacy rpyc files only have slot 1")
return zlib.decompress(data)
pos = len(RPYC2_HEADER)
while True:
header_slot, start, length = struct.unpack("III", data[pos : pos + 12])
if header_slot == slot:
return zlib.decompress(data[start : start + length])
if header_slot == 0:
raise ValueError("slot {0} not found".format(slot))
pos += 12
def py_string(value):
if value is None:
return None
if isinstance(value, bytes):
return value.decode("utf-8", "replace")
return text_type(value)
def simple_value(value):
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, (str, bytes)):
return py_string(value)
if isinstance(value, tuple):
return [simple_value(item) for item in value]
if isinstance(value, list):
return [simple_value(item) for item in value]
return py_string(value)
def node_fields(node):
kind = type(node).__name__
if kind == "Label":
return {"name": py_string(node.name)}
if kind == "Say":
return {
"who": py_string(node.who),
"what": py_string(node.what),
"identifier": py_string(getattr(node, "identifier", None)),
}
if kind == "Jump":
return {"target": py_string(node.target), "expression": bool(node.expression)}
if kind == "Call":
return {"label": py_string(node.label), "expression": bool(node.expression)}
if kind == "If":
return {"conditions": [simple_value(condition) for condition, _block in node.entries]}
if kind == "Menu":
return {
"items": [
{"label": py_string(label), "condition": simple_value(condition), "has_block": block is not None}
for label, condition, block in node.items
]
}
if kind == "Python":
return {"source": py_string(node.code.source), "store": py_string(node.store), "hide": bool(node.hide)}
return {}
def flatten(stmts):
nodes = []
for stmt in stmts:
stmt.get_children(nodes.append)
return nodes
def main():
args = parse_args()
renpy_root = os.path.abspath(args.renpy_root)
sys.path.insert(0, renpy_root)
import renpy # pylint: disable=import-error,import-outside-toplevel
import renpy.object # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.ast # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.python # pylint: disable=import-error,unused-import,import-outside-toplevel
install_minimal_renpy_runtime(renpy)
payload = pickle.loads(read_rpyc_slot(args.rpyc, args.slot))
metadata, stmts = payload
nodes = flatten(stmts)
line_counts = Counter((py_string(node.filename), int(node.linenumber)) for node in nodes)
seen = Counter()
records = []
for node in nodes:
path = py_string(node.filename)
line = int(node.linenumber)
occurrence = seen[(path, line)]
seen[(path, line)] += 1
records.append(
{
"key": {
"namespace": args.namespace,
"path": path,
"line": line,
"occurrence": occurrence,
},
"kind": type(node).__name__,
"fields": node_fields(node),
}
)
export = {
"schema": "ariadne-renpy-ast-adapter-prototype-v1",
"namespace": args.namespace,
"renpy_version": py_string(renpy.version),
"script_version": metadata.get("version"),
"slot": args.slot,
"source": os.path.abspath(args.rpyc),
"node_count": len(records),
"ambiguous_source_locations": sum(1 for count in line_counts.values() if count > 1),
"records": records,
}
output = os.path.abspath(args.output)
parent = os.path.dirname(output)
if parent and not os.path.isdir(parent):
os.makedirs(parent)
with open(output, "w") as handle:
json.dump(export, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps({key: export[key] for key in ("renpy_version", "script_version", "node_count", "ambiguous_source_locations")}, indent=2, sort_keys=True))
if __name__ == "__main__":
main()

49
src/ariadne/prototypes/renpy-ast-adapter/readme.md Normal file
View File

@@ -0,0 +1,49 @@
# Ren'Py AST adapter prototype
This throwaway prototype tests option 2 from the Ariadne design discussion: use the target launcher's own Python and Ren'Py modules to load Ren'Py AST pickle data, export normalized JSON records, and let Rust import those records into Ariadne-shaped state.
The prototype keeps Ren'Py internals outside Rust. Rust consumes only adapter JSON.
## Shape tested
```text
.rpyc slot
-> bundled Python adapter
-> normalized AST JSON records
-> Rust State { nodes, edges, pending }
```
The Rust state mirrors the current design sketch:
```rust
State {
nodes: Map<NodeKey, Node>,
edges: Map<NodeKey, Set<NodeKey>>,
pending: Vec<Operation>,
}
```
## Run
Run the adapter with the Python interpreter bundled with the target launcher. On NixOS, wrap that interpreter with `steam-run`.
```sh
steam-run /path/to/launcher/lib/linux-x86_64/python \
prototypes/renpy-ast-adapter/export_renpy_ast.py \
--renpy-root /path/to/launcher \
--namespace example \
--rpyc /path/to/script.rpyc \
--slot 1 \
--output /tmp/ariadne-renpy-ast-export.json
cargo run --manifest-path prototypes/renpy-ast-adapter/Cargo.toml -- \
/tmp/ariadne-renpy-ast-export.json
```
Use local ignored paths for target-game files. Do not commit generated exports.
## Early observation
Ren'Py AST can contain multiple nodes with the same source file and line. Examples include transition/display nodes sharing a physical source line, and slot 2 translation wrappers around `Say` nodes.
The prototype therefore adds a per-source-location `occurrence` field to `NodeKey`. This is evidence for revisiting whether Ariadne's final source key needs a similar disambiguator when AST ingestion is used.

163
src/ariadne/prototypes/renpy-ast-adapter/src/main.rs Normal file
View File

@@ -0,0 +1,163 @@
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::fs;
use anyhow::{Context, Result};
use serde::Deserialize;
use serde_json::Value;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize)]
struct NodeKey {
namespace: String,
path: String,
line: u32,
occurrence: u32,
}
#[derive(Debug, Clone)]
struct Node {
kind: String,
fields: Value,
}
#[derive(Debug)]
struct State {
nodes: BTreeMap<NodeKey, Node>,
edges: BTreeMap<NodeKey, BTreeSet<NodeKey>>,
pending: Vec<Operation>,
}
#[derive(Debug)]
enum Operation {
AddNode { key: NodeKey, node: Node },
}
#[derive(Debug, Deserialize)]
struct ExportFile {
schema: String,
namespace: String,
renpy_version: String,
script_version: u64,
slot: u8,
source: String,
node_count: usize,
ambiguous_source_locations: usize,
records: Vec<Record>,
}
#[derive(Debug, Deserialize)]
struct Record {
key: NodeKey,
kind: String,
fields: Value,
}
impl State {
fn empty() -> Self {
Self {
nodes: BTreeMap::new(),
edges: BTreeMap::new(),
pending: Vec::new(),
}
}
fn queue(&mut self, operation: Operation) {
self.pending.push(operation);
}
fn apply_pending(&mut self) {
for operation in self.pending.drain(..) {
match operation {
Operation::AddNode { key, node } => {
self.nodes.insert(key, node);
}
}
}
}
}
fn import_export(state: &mut State, export: ExportFile) {
for record in export.records {
state.queue(Operation::AddNode {
key: record.key,
node: Node {
kind: record.kind,
fields: record.fields,
},
});
}
}
fn main() -> Result<()> {
let path = env::args()
.nth(1)
.context("usage: renpy-ast-adapter-prototype <renpy-ast-export.json>")?;
let text = fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
let export: ExportFile = serde_json::from_str(&text).with_context(|| format!("parse {path}"))?;
println!("schema: {}", export.schema);
println!("namespace: {}", export.namespace);
println!("renpy version: {}", export.renpy_version);
println!("script version: {}", export.script_version);
println!("slot: {}", export.slot);
println!("source: {}", export.source);
println!("exported nodes: {}", export.node_count);
println!(
"source locations with multiple AST nodes: {}",
export.ambiguous_source_locations
);
let mut state = State::empty();
import_export(&mut state, export);
println!("pending operations before apply: {}", state.pending.len());
state.apply_pending();
println!("state nodes after apply: {}", state.nodes.len());
println!("state edge sources after apply: {}", state.edges.len());
let mut by_kind: BTreeMap<&str, usize> = BTreeMap::new();
for node in state.nodes.values() {
*by_kind.entry(node.kind.as_str()).or_default() += 1;
}
println!("nodes by kind:");
for (kind, count) in by_kind {
println!(" {kind}: {count}");
}
println!("first nodes:");
for (key, node) in state.nodes.iter().take(20) {
println!(
" {}:{}#{} {} {}",
key.path,
key.line,
key.occurrence,
node.kind,
compact_fields(&node.fields)
);
}
Ok(())
}
fn compact_fields(fields: &Value) -> String {
let Some(object) = fields.as_object() else {
return String::new();
};
let mut parts = Vec::new();
for (key, value) in object.iter().take(3) {
if value.is_null() {
continue;
}
parts.push(format!("{key}={}", short_json(value)));
}
parts.join(" ")
}
fn short_json(value: &Value) -> String {
let mut text = value.to_string();
const LIMIT: usize = 80;
if text.len() > LIMIT {
text.truncate(LIMIT);
text.push('…');
}
text
}

191
src/ariadne/tools/export-renpy-ast.py Executable file
View File

@@ -0,0 +1,191 @@
#!/usr/bin/env python
"""Export a Ren'Py AST pickle to Ariadne-neutral JSON records.
Run this with the Python interpreter bundled with the target Ren'Py launcher.
The script intentionally emits a small normalized record set instead of exposing
Ren'Py objects to Rust.
"""
from __future__ import print_function
import argparse
import json
import os
import pickle
import struct
import sys
import zlib
from collections import Counter
try:
text_type = unicode # noqa: F821 - Python 2 runtime.
except NameError:
text_type = str
RPYC2_HEADER = b"RENPY RPC2"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--renpy-root", required=True, help="Launcher root containing the renpy package")
parser.add_argument("--namespace", required=True, help="Namespace assigned to exported nodes")
parser.add_argument("--rpyc", required=True, help="Compiled Ren'Py script file")
parser.add_argument("--slot", type=int, default=1, choices=(1, 2), help="RPYC slot to export")
parser.add_argument("--output", required=True, help="JSON output path")
return parser.parse_args()
def install_minimal_renpy_runtime(renpy):
class Obj(object):
pass
script = Obj()
script.all_pyexpr = []
script.all_pycode = []
script.record_pycode = False
log = Obj()
log.mutated = {}
game = Obj()
game.script = script
game.log = log
renpy.game = game
def read_rpyc_slot(path, slot):
with open(path, "rb") as handle:
data = handle.read()
if not data.startswith(RPYC2_HEADER):
if slot != 1:
raise ValueError("legacy rpyc files only have slot 1")
return zlib.decompress(data)
pos = len(RPYC2_HEADER)
while True:
header_slot, start, length = struct.unpack("III", data[pos : pos + 12])
if header_slot == slot:
return zlib.decompress(data[start : start + length])
if header_slot == 0:
raise ValueError("slot {0} not found".format(slot))
pos += 12
def py_string(value):
if value is None:
return None
if isinstance(value, bytes):
return value.decode("utf-8", "replace")
return text_type(value)
def simple_value(value):
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, (str, bytes)):
return py_string(value)
if isinstance(value, tuple):
return [simple_value(item) for item in value]
if isinstance(value, list):
return [simple_value(item) for item in value]
return py_string(value)
def node_fields(node):
kind = type(node).__name__
if kind == "Label":
return {"name": py_string(node.name)}
if kind == "Say":
return {
"who": py_string(node.who),
"what": py_string(node.what),
"identifier": py_string(getattr(node, "identifier", None)),
}
if kind == "Jump":
return {"target": py_string(node.target), "expression": bool(node.expression)}
if kind == "Call":
return {"label": py_string(node.label), "expression": bool(node.expression)}
if kind == "If":
return {"conditions": [simple_value(condition) for condition, _block in node.entries]}
if kind == "Menu":
return {
"items": [
{"label": py_string(label), "condition": simple_value(condition), "has_block": block is not None}
for label, condition, block in node.items
]
}
if kind == "Python":
return {"source": py_string(node.code.source), "store": py_string(node.store), "hide": bool(node.hide)}
return {}
def flatten(stmts):
nodes = []
for stmt in stmts:
stmt.get_children(nodes.append)
return nodes
def main():
args = parse_args()
renpy_root = os.path.abspath(args.renpy_root)
sys.path.insert(0, renpy_root)
import renpy # pylint: disable=import-error,import-outside-toplevel
import renpy.object # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.ast # pylint: disable=import-error,unused-import,import-outside-toplevel
import renpy.python # pylint: disable=import-error,unused-import,import-outside-toplevel
install_minimal_renpy_runtime(renpy)
payload = pickle.loads(read_rpyc_slot(args.rpyc, args.slot))
metadata, stmts = payload
nodes = flatten(stmts)
line_counts = Counter((py_string(node.filename), int(node.linenumber)) for node in nodes)
seen = Counter()
records = []
for node in nodes:
path = py_string(node.filename)
line = int(node.linenumber)
occurrence = seen[(path, line)]
seen[(path, line)] += 1
records.append(
{
"key": {
"namespace": args.namespace,
"path": path,
"line": line,
"occurrence": occurrence,
},
"kind": type(node).__name__,
"fields": node_fields(node),
}
)
export = {
"schema": "ariadne-renpy-ast-adapter-prototype-v1",
"namespace": args.namespace,
"renpy_version": py_string(renpy.version),
"script_version": metadata.get("version"),
"slot": args.slot,
"source": os.path.abspath(args.rpyc),
"node_count": len(records),
"ambiguous_source_locations": sum(1 for count in line_counts.values() if count > 1),
"records": records,
}
output = os.path.abspath(args.output)
parent = os.path.dirname(output)
if parent and not os.path.isdir(parent):
os.makedirs(parent)
with open(output, "w") as handle:
json.dump(export, handle, indent=2, sort_keys=True)
handle.write("\n")
print(json.dumps({key: export[key] for key in ("renpy_version", "script_version", "node_count", "ambiguous_source_locations")}, indent=2, sort_keys=True))
if __name__ == "__main__":
main()