feat(ariadne): export Ren'Py AST artifacts
This commit is contained in:
@@ -7,11 +7,11 @@ Usage: extract-and-decompile --game-root <path> [--output-root <path>] [--force]
|
|||||||
|
|
||||||
Prepare local game-owned source for Ariadne.
|
Prepare local game-owned source for Ariadne.
|
||||||
|
|
||||||
Use the game installation directory as the game root to process all installed seasons.
|
Use the game installation directory as the game root to process all installed launchers.
|
||||||
The utility writes recovered source under <output-root>/source/, transient work under <output-root>/work/, and local logs under <output-root>/logs/.
|
The utility writes recovered source under <output-root>/source/, normalized Ren'Py AST exports under <output-root>/ast/, transient work under <output-root>/work/, and local logs under <output-root>/logs/.
|
||||||
The default output root is <workspace>/src/ariadne/output.
|
The default output root is <workspace>/src/ariadne/output.
|
||||||
Without --force, it refuses to run when any of those directories already exists.
|
Without --force, it refuses to run when any of those directories already exists.
|
||||||
With --force, it deletes and recreates only source/, work/, and logs/ under the output root.
|
With --force, it deletes and recreates only source/, ast/, work/, and logs/ under the output root.
|
||||||
The script enters a Nix shell for rpycdec when required tools are not already available.
|
The script enters a Nix shell for rpycdec when required tools are not already available.
|
||||||
Game-derived preparation output must remain local and must not be committed.
|
Game-derived preparation output must remain local and must not be committed.
|
||||||
USAGE
|
USAGE
|
||||||
@@ -109,7 +109,7 @@ done
|
|||||||
|
|
||||||
enter_nix_shell_if_needed "${original_args[@]}"
|
enter_nix_shell_if_needed "${original_args[@]}"
|
||||||
|
|
||||||
preparation_dirs=(source work logs)
|
preparation_dirs=(source ast work logs)
|
||||||
|
|
||||||
for name in "${preparation_dirs[@]}"; do
|
for name in "${preparation_dirs[@]}"; do
|
||||||
path="$output_root/$name"
|
path="$output_root/$name"
|
||||||
@@ -131,6 +131,7 @@ for name in "${preparation_dirs[@]}"; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
source_dir=$output_root/source
|
source_dir=$output_root/source
|
||||||
|
ast_dir=$output_root/ast
|
||||||
work_dir=$output_root/work
|
work_dir=$output_root/work
|
||||||
logs_dir=$output_root/logs
|
logs_dir=$output_root/logs
|
||||||
summary_log=$logs_dir/extract-and-decompile.log
|
summary_log=$logs_dir/extract-and-decompile.log
|
||||||
@@ -174,6 +175,43 @@ source_log_relative() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
launcher_prefix() {
|
||||||
|
local log_relative=$1
|
||||||
|
printf '%s\n' "${log_relative%%/*}"
|
||||||
|
}
|
||||||
|
|
||||||
|
launcher_root_for() {
|
||||||
|
local log_relative=$1
|
||||||
|
local prefix
|
||||||
|
prefix=$(launcher_prefix "$log_relative")
|
||||||
|
printf '%s/%s\n' "$game_root" "$prefix"
|
||||||
|
}
|
||||||
|
|
||||||
|
bundled_python_for() {
|
||||||
|
local launcher_root=$1
|
||||||
|
local candidate
|
||||||
|
for candidate in \
|
||||||
|
"$launcher_root/lib/linux-x86_64/python" \
|
||||||
|
"$launcher_root/lib/linux-i686/python"; do
|
||||||
|
if [[ -x "$candidate" ]]; then
|
||||||
|
printf '%s\n' "$candidate"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fail "no bundled Linux Python found under launcher root: $launcher_root"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_bundled_python() {
|
||||||
|
local python=$1
|
||||||
|
shift
|
||||||
|
python=$(realpath "$python")
|
||||||
|
if command -v steam-run >/dev/null 2>&1; then
|
||||||
|
steam-run "$python" "$@"
|
||||||
|
else
|
||||||
|
"$python" "$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
copy_source_file() {
|
copy_source_file() {
|
||||||
local input=$1
|
local input=$1
|
||||||
local input_root=$2
|
local input_root=$2
|
||||||
@@ -196,6 +234,28 @@ copy_loose_source() {
|
|||||||
copy_source_file "$1" "$game_root" loose
|
copy_source_file "$1" "$game_root" loose
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export_ast() {
|
||||||
|
local input=$1
|
||||||
|
local log_relative=$2
|
||||||
|
local launcher_root python adapter output log_file namespace
|
||||||
|
launcher_root=$(launcher_root_for "$log_relative")
|
||||||
|
python=$(bundled_python_for "$launcher_root")
|
||||||
|
adapter=$script_dir/tools/export-renpy-ast.py
|
||||||
|
namespace=$(launcher_prefix "$log_relative")
|
||||||
|
output=$ast_dir/${log_relative%.*}.json
|
||||||
|
log_file=$logs_dir/ast-${log_relative//\//__}.log
|
||||||
|
[[ -f "$adapter" ]] || fail "missing Ren'Py AST adapter: $adapter"
|
||||||
|
mkdir -p -- "$(dirname -- "$output")"
|
||||||
|
say "exporting Ren'Py AST: $log_relative"
|
||||||
|
run_bundled_python "$python" "$adapter" \
|
||||||
|
--renpy-root "$launcher_root" \
|
||||||
|
--namespace "$namespace" \
|
||||||
|
--rpyc "$input" \
|
||||||
|
--slot 1 \
|
||||||
|
--output "$output" > "$log_file" 2>&1
|
||||||
|
log "exported Ren'Py AST: $log_relative"
|
||||||
|
}
|
||||||
|
|
||||||
decompile_compiled_script() {
|
decompile_compiled_script() {
|
||||||
local input=$1
|
local input=$1
|
||||||
local input_root=$2
|
local input_root=$2
|
||||||
@@ -207,6 +267,7 @@ decompile_compiled_script() {
|
|||||||
log "skipped Ren'Py runtime compiled script: $log_relative"
|
log "skipped Ren'Py runtime compiled script: $log_relative"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
export_ast "$input" "$log_relative"
|
||||||
output_dir=$(source_output_path "$(dirname -- "$relative")" "$destination_prefix")
|
output_dir=$(source_output_path "$(dirname -- "$relative")" "$destination_prefix")
|
||||||
log_file=$logs_dir/decompile-${log_relative//\//__}.log
|
log_file=$logs_dir/decompile-${log_relative//\//__}.log
|
||||||
mkdir -p -- "$output_dir"
|
mkdir -p -- "$output_dir"
|
||||||
|
|||||||
114
src/ariadne/prototypes/renpy-ast-adapter/Cargo.lock
generated
Normal file
114
src/ariadne/prototypes/renpy-ast-adapter/Cargo.lock
generated
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anyhow"
|
||||||
|
version = "1.0.104"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.107"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "renpy-ast-adapter-prototype"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.151"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "3.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.24"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
11
src/ariadne/prototypes/renpy-ast-adapter/Cargo.toml
Normal file
11
src/ariadne/prototypes/renpy-ast-adapter/Cargo.toml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[package]
|
||||||
|
name = "renpy-ast-adapter-prototype"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[workspace]
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1.0"
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0"
|
||||||
191
src/ariadne/prototypes/renpy-ast-adapter/export_renpy_ast.py
Normal file
191
src/ariadne/prototypes/renpy-ast-adapter/export_renpy_ast.py
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
"""Export a Ren'Py AST pickle to Ariadne-neutral JSON records.
|
||||||
|
|
||||||
|
Run this with the Python interpreter bundled with the target Ren'Py launcher.
|
||||||
|
The script intentionally emits a small normalized record set instead of exposing
|
||||||
|
Ren'Py objects to Rust.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pickle
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
import zlib
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
try:
|
||||||
|
text_type = unicode # noqa: F821 - Python 2 runtime.
|
||||||
|
except NameError:
|
||||||
|
text_type = str
|
||||||
|
|
||||||
|
RPYC2_HEADER = b"RENPY RPC2"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--renpy-root", required=True, help="Launcher root containing the renpy package")
|
||||||
|
parser.add_argument("--namespace", required=True, help="Namespace assigned to exported nodes")
|
||||||
|
parser.add_argument("--rpyc", required=True, help="Compiled Ren'Py script file")
|
||||||
|
parser.add_argument("--slot", type=int, default=1, choices=(1, 2), help="RPYC slot to export")
|
||||||
|
parser.add_argument("--output", required=True, help="JSON output path")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def install_minimal_renpy_runtime(renpy):
|
||||||
|
class Obj(object):
|
||||||
|
pass
|
||||||
|
|
||||||
|
script = Obj()
|
||||||
|
script.all_pyexpr = []
|
||||||
|
script.all_pycode = []
|
||||||
|
script.record_pycode = False
|
||||||
|
|
||||||
|
log = Obj()
|
||||||
|
log.mutated = {}
|
||||||
|
|
||||||
|
game = Obj()
|
||||||
|
game.script = script
|
||||||
|
game.log = log
|
||||||
|
renpy.game = game
|
||||||
|
|
||||||
|
|
||||||
|
def read_rpyc_slot(path, slot):
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
data = handle.read()
|
||||||
|
|
||||||
|
if not data.startswith(RPYC2_HEADER):
|
||||||
|
if slot != 1:
|
||||||
|
raise ValueError("legacy rpyc files only have slot 1")
|
||||||
|
return zlib.decompress(data)
|
||||||
|
|
||||||
|
pos = len(RPYC2_HEADER)
|
||||||
|
while True:
|
||||||
|
header_slot, start, length = struct.unpack("III", data[pos : pos + 12])
|
||||||
|
if header_slot == slot:
|
||||||
|
return zlib.decompress(data[start : start + length])
|
||||||
|
if header_slot == 0:
|
||||||
|
raise ValueError("slot {0} not found".format(slot))
|
||||||
|
pos += 12
|
||||||
|
|
||||||
|
|
||||||
|
def py_string(value):
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.decode("utf-8", "replace")
|
||||||
|
return text_type(value)
|
||||||
|
|
||||||
|
|
||||||
|
def simple_value(value):
|
||||||
|
if value is None or isinstance(value, (bool, int, float)):
|
||||||
|
return value
|
||||||
|
if isinstance(value, (str, bytes)):
|
||||||
|
return py_string(value)
|
||||||
|
if isinstance(value, tuple):
|
||||||
|
return [simple_value(item) for item in value]
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [simple_value(item) for item in value]
|
||||||
|
return py_string(value)
|
||||||
|
|
||||||
|
|
||||||
|
def node_fields(node):
|
||||||
|
kind = type(node).__name__
|
||||||
|
if kind == "Label":
|
||||||
|
return {"name": py_string(node.name)}
|
||||||
|
if kind == "Say":
|
||||||
|
return {
|
||||||
|
"who": py_string(node.who),
|
||||||
|
"what": py_string(node.what),
|
||||||
|
"identifier": py_string(getattr(node, "identifier", None)),
|
||||||
|
}
|
||||||
|
if kind == "Jump":
|
||||||
|
return {"target": py_string(node.target), "expression": bool(node.expression)}
|
||||||
|
if kind == "Call":
|
||||||
|
return {"label": py_string(node.label), "expression": bool(node.expression)}
|
||||||
|
if kind == "If":
|
||||||
|
return {"conditions": [simple_value(condition) for condition, _block in node.entries]}
|
||||||
|
if kind == "Menu":
|
||||||
|
return {
|
||||||
|
"items": [
|
||||||
|
{"label": py_string(label), "condition": simple_value(condition), "has_block": block is not None}
|
||||||
|
for label, condition, block in node.items
|
||||||
|
]
|
||||||
|
}
|
||||||
|
if kind == "Python":
|
||||||
|
return {"source": py_string(node.code.source), "store": py_string(node.store), "hide": bool(node.hide)}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def flatten(stmts):
|
||||||
|
nodes = []
|
||||||
|
for stmt in stmts:
|
||||||
|
stmt.get_children(nodes.append)
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
renpy_root = os.path.abspath(args.renpy_root)
|
||||||
|
sys.path.insert(0, renpy_root)
|
||||||
|
|
||||||
|
import renpy # pylint: disable=import-error,import-outside-toplevel
|
||||||
|
import renpy.object # pylint: disable=import-error,unused-import,import-outside-toplevel
|
||||||
|
import renpy.ast # pylint: disable=import-error,unused-import,import-outside-toplevel
|
||||||
|
import renpy.python # pylint: disable=import-error,unused-import,import-outside-toplevel
|
||||||
|
|
||||||
|
install_minimal_renpy_runtime(renpy)
|
||||||
|
|
||||||
|
payload = pickle.loads(read_rpyc_slot(args.rpyc, args.slot))
|
||||||
|
metadata, stmts = payload
|
||||||
|
nodes = flatten(stmts)
|
||||||
|
|
||||||
|
line_counts = Counter((py_string(node.filename), int(node.linenumber)) for node in nodes)
|
||||||
|
seen = Counter()
|
||||||
|
records = []
|
||||||
|
for node in nodes:
|
||||||
|
path = py_string(node.filename)
|
||||||
|
line = int(node.linenumber)
|
||||||
|
occurrence = seen[(path, line)]
|
||||||
|
seen[(path, line)] += 1
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"key": {
|
||||||
|
"namespace": args.namespace,
|
||||||
|
"path": path,
|
||||||
|
"line": line,
|
||||||
|
"occurrence": occurrence,
|
||||||
|
},
|
||||||
|
"kind": type(node).__name__,
|
||||||
|
"fields": node_fields(node),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
export = {
|
||||||
|
"schema": "ariadne-renpy-ast-adapter-prototype-v1",
|
||||||
|
"namespace": args.namespace,
|
||||||
|
"renpy_version": py_string(renpy.version),
|
||||||
|
"script_version": metadata.get("version"),
|
||||||
|
"slot": args.slot,
|
||||||
|
"source": os.path.abspath(args.rpyc),
|
||||||
|
"node_count": len(records),
|
||||||
|
"ambiguous_source_locations": sum(1 for count in line_counts.values() if count > 1),
|
||||||
|
"records": records,
|
||||||
|
}
|
||||||
|
|
||||||
|
output = os.path.abspath(args.output)
|
||||||
|
parent = os.path.dirname(output)
|
||||||
|
if parent and not os.path.isdir(parent):
|
||||||
|
os.makedirs(parent)
|
||||||
|
with open(output, "w") as handle:
|
||||||
|
json.dump(export, handle, indent=2, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
|
||||||
|
print(json.dumps({key: export[key] for key in ("renpy_version", "script_version", "node_count", "ambiguous_source_locations")}, indent=2, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
49
src/ariadne/prototypes/renpy-ast-adapter/readme.md
Normal file
49
src/ariadne/prototypes/renpy-ast-adapter/readme.md
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
# Ren'Py AST adapter prototype
|
||||||
|
|
||||||
|
This throwaway prototype tests option 2 from the Ariadne design discussion: use the target launcher's own Python and Ren'Py modules to load Ren'Py AST pickle data, export normalized JSON records, and let Rust import those records into Ariadne-shaped state.
|
||||||
|
|
||||||
|
The prototype keeps Ren'Py internals outside Rust. Rust consumes only adapter JSON.
|
||||||
|
|
||||||
|
## Shape tested
|
||||||
|
|
||||||
|
```text
|
||||||
|
.rpyc slot
|
||||||
|
-> bundled Python adapter
|
||||||
|
-> normalized AST JSON records
|
||||||
|
-> Rust State { nodes, edges, pending }
|
||||||
|
```
|
||||||
|
|
||||||
|
The Rust state mirrors the current design sketch:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
State {
|
||||||
|
nodes: Map<NodeKey, Node>,
|
||||||
|
edges: Map<NodeKey, Set<NodeKey>>,
|
||||||
|
pending: Vec<Operation>,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Run
|
||||||
|
|
||||||
|
Run the adapter with the Python interpreter bundled with the target launcher. On NixOS, wrap that interpreter with `steam-run`.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
steam-run /path/to/launcher/lib/linux-x86_64/python \
|
||||||
|
prototypes/renpy-ast-adapter/export_renpy_ast.py \
|
||||||
|
--renpy-root /path/to/launcher \
|
||||||
|
--namespace example \
|
||||||
|
--rpyc /path/to/script.rpyc \
|
||||||
|
--slot 1 \
|
||||||
|
--output /tmp/ariadne-renpy-ast-export.json
|
||||||
|
|
||||||
|
cargo run --manifest-path prototypes/renpy-ast-adapter/Cargo.toml -- \
|
||||||
|
/tmp/ariadne-renpy-ast-export.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Use local ignored paths for target-game files. Do not commit generated exports.
|
||||||
|
|
||||||
|
## Early observation
|
||||||
|
|
||||||
|
Ren'Py AST can contain multiple nodes with the same source file and line. Examples include transition/display nodes sharing a physical source line, and slot 2 translation wrappers around `Say` nodes.
|
||||||
|
|
||||||
|
The prototype therefore adds a per-source-location `occurrence` field to `NodeKey`. This is evidence for revisiting whether Ariadne's final source key needs a similar disambiguator when AST ingestion is used.
|
||||||
163
src/ariadne/prototypes/renpy-ast-adapter/src/main.rs
Normal file
163
src/ariadne/prototypes/renpy-ast-adapter/src/main.rs
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
use std::env;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize)]
|
||||||
|
struct NodeKey {
|
||||||
|
namespace: String,
|
||||||
|
path: String,
|
||||||
|
line: u32,
|
||||||
|
occurrence: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct Node {
|
||||||
|
kind: String,
|
||||||
|
fields: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct State {
|
||||||
|
nodes: BTreeMap<NodeKey, Node>,
|
||||||
|
edges: BTreeMap<NodeKey, BTreeSet<NodeKey>>,
|
||||||
|
pending: Vec<Operation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum Operation {
|
||||||
|
AddNode { key: NodeKey, node: Node },
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct ExportFile {
|
||||||
|
schema: String,
|
||||||
|
namespace: String,
|
||||||
|
renpy_version: String,
|
||||||
|
script_version: u64,
|
||||||
|
slot: u8,
|
||||||
|
source: String,
|
||||||
|
node_count: usize,
|
||||||
|
ambiguous_source_locations: usize,
|
||||||
|
records: Vec<Record>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct Record {
|
||||||
|
key: NodeKey,
|
||||||
|
kind: String,
|
||||||
|
fields: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl State {
|
||||||
|
fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
nodes: BTreeMap::new(),
|
||||||
|
edges: BTreeMap::new(),
|
||||||
|
pending: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn queue(&mut self, operation: Operation) {
|
||||||
|
self.pending.push(operation);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_pending(&mut self) {
|
||||||
|
for operation in self.pending.drain(..) {
|
||||||
|
match operation {
|
||||||
|
Operation::AddNode { key, node } => {
|
||||||
|
self.nodes.insert(key, node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn import_export(state: &mut State, export: ExportFile) {
|
||||||
|
for record in export.records {
|
||||||
|
state.queue(Operation::AddNode {
|
||||||
|
key: record.key,
|
||||||
|
node: Node {
|
||||||
|
kind: record.kind,
|
||||||
|
fields: record.fields,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
let path = env::args()
|
||||||
|
.nth(1)
|
||||||
|
.context("usage: renpy-ast-adapter-prototype <renpy-ast-export.json>")?;
|
||||||
|
let text = fs::read_to_string(&path).with_context(|| format!("read {path}"))?;
|
||||||
|
let export: ExportFile = serde_json::from_str(&text).with_context(|| format!("parse {path}"))?;
|
||||||
|
|
||||||
|
println!("schema: {}", export.schema);
|
||||||
|
println!("namespace: {}", export.namespace);
|
||||||
|
println!("renpy version: {}", export.renpy_version);
|
||||||
|
println!("script version: {}", export.script_version);
|
||||||
|
println!("slot: {}", export.slot);
|
||||||
|
println!("source: {}", export.source);
|
||||||
|
println!("exported nodes: {}", export.node_count);
|
||||||
|
println!(
|
||||||
|
"source locations with multiple AST nodes: {}",
|
||||||
|
export.ambiguous_source_locations
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut state = State::empty();
|
||||||
|
import_export(&mut state, export);
|
||||||
|
println!("pending operations before apply: {}", state.pending.len());
|
||||||
|
state.apply_pending();
|
||||||
|
println!("state nodes after apply: {}", state.nodes.len());
|
||||||
|
println!("state edge sources after apply: {}", state.edges.len());
|
||||||
|
|
||||||
|
let mut by_kind: BTreeMap<&str, usize> = BTreeMap::new();
|
||||||
|
for node in state.nodes.values() {
|
||||||
|
*by_kind.entry(node.kind.as_str()).or_default() += 1;
|
||||||
|
}
|
||||||
|
println!("nodes by kind:");
|
||||||
|
for (kind, count) in by_kind {
|
||||||
|
println!(" {kind}: {count}");
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("first nodes:");
|
||||||
|
for (key, node) in state.nodes.iter().take(20) {
|
||||||
|
println!(
|
||||||
|
" {}:{}#{} {} {}",
|
||||||
|
key.path,
|
||||||
|
key.line,
|
||||||
|
key.occurrence,
|
||||||
|
node.kind,
|
||||||
|
compact_fields(&node.fields)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compact_fields(fields: &Value) -> String {
|
||||||
|
let Some(object) = fields.as_object() else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
|
let mut parts = Vec::new();
|
||||||
|
for (key, value) in object.iter().take(3) {
|
||||||
|
if value.is_null() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
parts.push(format!("{key}={}", short_json(value)));
|
||||||
|
}
|
||||||
|
parts.join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn short_json(value: &Value) -> String {
|
||||||
|
let mut text = value.to_string();
|
||||||
|
const LIMIT: usize = 80;
|
||||||
|
if text.len() > LIMIT {
|
||||||
|
text.truncate(LIMIT);
|
||||||
|
text.push('…');
|
||||||
|
}
|
||||||
|
text
|
||||||
|
}
|
||||||
191
src/ariadne/tools/export-renpy-ast.py
Executable file
191
src/ariadne/tools/export-renpy-ast.py
Executable file
@@ -0,0 +1,191 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
"""Export a Ren'Py AST pickle to Ariadne-neutral JSON records.
|
||||||
|
|
||||||
|
Run this with the Python interpreter bundled with the target Ren'Py launcher.
|
||||||
|
The script intentionally emits a small normalized record set instead of exposing
|
||||||
|
Ren'Py objects to Rust.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pickle
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
import zlib
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
try:
|
||||||
|
text_type = unicode # noqa: F821 - Python 2 runtime.
|
||||||
|
except NameError:
|
||||||
|
text_type = str
|
||||||
|
|
||||||
|
RPYC2_HEADER = b"RENPY RPC2"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--renpy-root", required=True, help="Launcher root containing the renpy package")
|
||||||
|
parser.add_argument("--namespace", required=True, help="Namespace assigned to exported nodes")
|
||||||
|
parser.add_argument("--rpyc", required=True, help="Compiled Ren'Py script file")
|
||||||
|
parser.add_argument("--slot", type=int, default=1, choices=(1, 2), help="RPYC slot to export")
|
||||||
|
parser.add_argument("--output", required=True, help="JSON output path")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def install_minimal_renpy_runtime(renpy):
|
||||||
|
class Obj(object):
|
||||||
|
pass
|
||||||
|
|
||||||
|
script = Obj()
|
||||||
|
script.all_pyexpr = []
|
||||||
|
script.all_pycode = []
|
||||||
|
script.record_pycode = False
|
||||||
|
|
||||||
|
log = Obj()
|
||||||
|
log.mutated = {}
|
||||||
|
|
||||||
|
game = Obj()
|
||||||
|
game.script = script
|
||||||
|
game.log = log
|
||||||
|
renpy.game = game
|
||||||
|
|
||||||
|
|
||||||
|
def read_rpyc_slot(path, slot):
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
data = handle.read()
|
||||||
|
|
||||||
|
if not data.startswith(RPYC2_HEADER):
|
||||||
|
if slot != 1:
|
||||||
|
raise ValueError("legacy rpyc files only have slot 1")
|
||||||
|
return zlib.decompress(data)
|
||||||
|
|
||||||
|
pos = len(RPYC2_HEADER)
|
||||||
|
while True:
|
||||||
|
header_slot, start, length = struct.unpack("III", data[pos : pos + 12])
|
||||||
|
if header_slot == slot:
|
||||||
|
return zlib.decompress(data[start : start + length])
|
||||||
|
if header_slot == 0:
|
||||||
|
raise ValueError("slot {0} not found".format(slot))
|
||||||
|
pos += 12
|
||||||
|
|
||||||
|
|
||||||
|
def py_string(value):
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.decode("utf-8", "replace")
|
||||||
|
return text_type(value)
|
||||||
|
|
||||||
|
|
||||||
|
def simple_value(value):
|
||||||
|
if value is None or isinstance(value, (bool, int, float)):
|
||||||
|
return value
|
||||||
|
if isinstance(value, (str, bytes)):
|
||||||
|
return py_string(value)
|
||||||
|
if isinstance(value, tuple):
|
||||||
|
return [simple_value(item) for item in value]
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [simple_value(item) for item in value]
|
||||||
|
return py_string(value)
|
||||||
|
|
||||||
|
|
||||||
|
def node_fields(node):
|
||||||
|
kind = type(node).__name__
|
||||||
|
if kind == "Label":
|
||||||
|
return {"name": py_string(node.name)}
|
||||||
|
if kind == "Say":
|
||||||
|
return {
|
||||||
|
"who": py_string(node.who),
|
||||||
|
"what": py_string(node.what),
|
||||||
|
"identifier": py_string(getattr(node, "identifier", None)),
|
||||||
|
}
|
||||||
|
if kind == "Jump":
|
||||||
|
return {"target": py_string(node.target), "expression": bool(node.expression)}
|
||||||
|
if kind == "Call":
|
||||||
|
return {"label": py_string(node.label), "expression": bool(node.expression)}
|
||||||
|
if kind == "If":
|
||||||
|
return {"conditions": [simple_value(condition) for condition, _block in node.entries]}
|
||||||
|
if kind == "Menu":
|
||||||
|
return {
|
||||||
|
"items": [
|
||||||
|
{"label": py_string(label), "condition": simple_value(condition), "has_block": block is not None}
|
||||||
|
for label, condition, block in node.items
|
||||||
|
]
|
||||||
|
}
|
||||||
|
if kind == "Python":
|
||||||
|
return {"source": py_string(node.code.source), "store": py_string(node.store), "hide": bool(node.hide)}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def flatten(stmts):
|
||||||
|
nodes = []
|
||||||
|
for stmt in stmts:
|
||||||
|
stmt.get_children(nodes.append)
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
renpy_root = os.path.abspath(args.renpy_root)
|
||||||
|
sys.path.insert(0, renpy_root)
|
||||||
|
|
||||||
|
import renpy # pylint: disable=import-error,import-outside-toplevel
|
||||||
|
import renpy.object # pylint: disable=import-error,unused-import,import-outside-toplevel
|
||||||
|
import renpy.ast # pylint: disable=import-error,unused-import,import-outside-toplevel
|
||||||
|
import renpy.python # pylint: disable=import-error,unused-import,import-outside-toplevel
|
||||||
|
|
||||||
|
install_minimal_renpy_runtime(renpy)
|
||||||
|
|
||||||
|
payload = pickle.loads(read_rpyc_slot(args.rpyc, args.slot))
|
||||||
|
metadata, stmts = payload
|
||||||
|
nodes = flatten(stmts)
|
||||||
|
|
||||||
|
line_counts = Counter((py_string(node.filename), int(node.linenumber)) for node in nodes)
|
||||||
|
seen = Counter()
|
||||||
|
records = []
|
||||||
|
for node in nodes:
|
||||||
|
path = py_string(node.filename)
|
||||||
|
line = int(node.linenumber)
|
||||||
|
occurrence = seen[(path, line)]
|
||||||
|
seen[(path, line)] += 1
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"key": {
|
||||||
|
"namespace": args.namespace,
|
||||||
|
"path": path,
|
||||||
|
"line": line,
|
||||||
|
"occurrence": occurrence,
|
||||||
|
},
|
||||||
|
"kind": type(node).__name__,
|
||||||
|
"fields": node_fields(node),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
export = {
|
||||||
|
"schema": "ariadne-renpy-ast-adapter-prototype-v1",
|
||||||
|
"namespace": args.namespace,
|
||||||
|
"renpy_version": py_string(renpy.version),
|
||||||
|
"script_version": metadata.get("version"),
|
||||||
|
"slot": args.slot,
|
||||||
|
"source": os.path.abspath(args.rpyc),
|
||||||
|
"node_count": len(records),
|
||||||
|
"ambiguous_source_locations": sum(1 for count in line_counts.values() if count > 1),
|
||||||
|
"records": records,
|
||||||
|
}
|
||||||
|
|
||||||
|
output = os.path.abspath(args.output)
|
||||||
|
parent = os.path.dirname(output)
|
||||||
|
if parent and not os.path.isdir(parent):
|
||||||
|
os.makedirs(parent)
|
||||||
|
with open(output, "w") as handle:
|
||||||
|
json.dump(export, handle, indent=2, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
|
||||||
|
print(json.dumps({key: export[key] for key in ("renpy_version", "script_version", "node_count", "ambiguous_source_locations")}, indent=2, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user