feat(ariadne): deserialize Clew AST JSON

src/ariadne/BUILD

@@ -4,9 +4,14 @@ rust_library(
     name = "ariadne_lib",
     crate_name = "ariadne",
     srcs = [
+        "clew.rs",
         "lib.rs",
         "pipeline.rs",
     ],
+    deps = [
+        "//third_party/crates:serde",
+        "//third_party/crates:serde_json",
+    ],
 )
 
 rust_binary(
@@ -19,6 +24,11 @@ rust_binary(
     ],
 )
 
+rust_test(
+    name = "ariadne_lib_test",
+    crate = ":ariadne_lib",
+)
+
 rust_test(
     name = "pipeline_test",
     srcs = ["pipeline_test.rs"],

src/ariadne/clew.rs

@@ -0,0 +1,435 @@
+use std::collections::BTreeMap;
+use std::error;
+use std::fmt;
+
+use serde::Deserialize;
+use serde_json::{Map, Number};
+
+const SCHEMA: &str = "clew-renpy-ast-v1";
+
+#[derive(Debug)]
+pub enum Error {
+    Json(serde_json::Error),
+    SchemaMismatch {
+        expected: &'static str,
+        actual: String,
+    },
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::Json(error) => write!(formatter, "invalid Clew JSON: {error}"),
+            Error::SchemaMismatch { expected, actual } => write!(
+                formatter,
+                "unsupported Clew schema: expected {expected:?}, got {actual:?}"
+            ),
+        }
+    }
+}
+
+impl error::Error for Error {
+    fn source(&self) -> Option<&(dyn error::Error + 'static)> {
+        match self {
+            Error::Json(error) => Some(error),
+            Error::SchemaMismatch { .. } => None,
+        }
+    }
+}
+
+impl From<serde_json::Error> for Error {
+    fn from(error: serde_json::Error) -> Self {
+        Error::Json(error)
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq)]
+pub struct Document {
+    pub schema: String,
+    pub source: String,
+    pub game_root: String,
+    pub slot: u64,
+    pub python_version: String,
+    pub renpy_version: Option<String>,
+    pub metadata: Value,
+    pub roots: Value,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum Value {
+    Null,
+    Bool(bool),
+    Number(Number),
+    String(String),
+    List(Vec<Value>),
+    Tuple(Vec<Value>),
+    Dict(Vec<DictEntry>),
+    Object(Object),
+    Ref { id: u64 },
+    Function(Function),
+    Method(Method),
+    PythonType(PythonType),
+    Repr { class_name: String, repr: String },
+    Bytes { repr: String },
+    Truncated { class_name: String, repr: String },
+    Recursion { class_name: String },
+    AttributeError { class_name: String, repr: String },
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct DictEntry {
+    pub key: Value,
+    pub value: Value,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Object {
+    pub id: u64,
+    pub class_name: String,
+    pub attributes: BTreeMap<String, Value>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Function {
+    pub name: Option<String>,
+    pub module: Option<String>,
+    pub filename: Option<String>,
+    pub firstlineno: Option<Number>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Method {
+    pub name: Option<String>,
+    pub module: Option<String>,
+    pub filename: Option<String>,
+    pub firstlineno: Option<Number>,
+    pub receiver_class: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct PythonType {
+    pub name: Option<String>,
+    pub module: Option<String>,
+}
+
+pub fn from_bytes(bytes: &[u8]) -> Result<Document, Error> {
+    let document: Document = serde_json::from_slice(bytes)?;
+    if document.schema != SCHEMA {
+        return Err(Error::SchemaMismatch {
+            expected: SCHEMA,
+            actual: document.schema,
+        });
+    }
+    Ok(document)
+}
+
+impl<'de> Deserialize<'de> for Value {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = serde_json::Value::deserialize(deserializer)?;
+        parse_value(value).map_err(serde::de::Error::custom)
+    }
+}
+
+fn parse_value(value: serde_json::Value) -> Result<Value, String> {
+    match value {
+        serde_json::Value::Null => Ok(Value::Null),
+        serde_json::Value::Bool(value) => Ok(Value::Bool(value)),
+        serde_json::Value::Number(value) => Ok(Value::Number(value)),
+        serde_json::Value::String(value) => Ok(Value::String(value)),
+        serde_json::Value::Array(_) => Err("Clew values do not use untagged arrays".to_string()),
+        serde_json::Value::Object(mut object) => parse_tagged_object(&mut object),
+    }
+}
+
+fn parse_tagged_object(object: &mut Map<String, serde_json::Value>) -> Result<Value, String> {
+    let tag = take_string(object, "type")?;
+    match tag.as_str() {
+        "list" => Ok(Value::List(take_items(object)?)),
+        "tuple" => Ok(Value::Tuple(take_items(object)?)),
+        "dict" => Ok(Value::Dict(take_dict_items(object)?)),
+        "object" => Ok(Value::Object(Object {
+            id: take_u64(object, "id")?,
+            class_name: take_string(object, "class")?,
+            attributes: take_attributes(object)?,
+        })),
+        "ref" => Ok(Value::Ref {
+            id: take_u64(object, "id")?,
+        }),
+        "function" => Ok(Value::Function(Function {
+            name: take_optional_string(object, "name")?,
+            module: take_optional_string(object, "module")?,
+            filename: take_optional_string(object, "filename")?,
+            firstlineno: take_optional_number(object, "firstlineno")?,
+        })),
+        "method" => Ok(Value::Method(Method {
+            name: take_optional_string(object, "name")?,
+            module: take_optional_string(object, "module")?,
+            filename: take_optional_string(object, "filename")?,
+            firstlineno: take_optional_number(object, "firstlineno")?,
+            receiver_class: take_optional_string(object, "receiver_class")?,
+        })),
+        "python-type" => Ok(Value::PythonType(PythonType {
+            name: take_optional_string(object, "name")?,
+            module: take_optional_string(object, "module")?,
+        })),
+        "repr" => Ok(Value::Repr {
+            class_name: take_string(object, "class")?,
+            repr: take_string(object, "repr")?,
+        }),
+        "bytes" => Ok(Value::Bytes {
+            repr: take_string(object, "repr")?,
+        }),
+        "truncated" => Ok(Value::Truncated {
+            class_name: take_string(object, "class")?,
+            repr: take_string(object, "repr")?,
+        }),
+        "recursion" => Ok(Value::Recursion {
+            class_name: take_string(object, "class")?,
+        }),
+        "attribute-error" => Ok(Value::AttributeError {
+            class_name: take_string(object, "class")?,
+            repr: take_string(object, "repr")?,
+        }),
+        _ => Err(format!("unknown Clew value type: {tag}")),
+    }
+}
+
+fn take_items(object: &mut Map<String, serde_json::Value>) -> Result<Vec<Value>, String> {
+    match object.remove("items") {
+        Some(serde_json::Value::Array(items)) => items.into_iter().map(parse_value).collect(),
+        Some(_) => Err("Clew items field must be an array".to_string()),
+        None => Err("missing Clew items field".to_string()),
+    }
+}
+
+fn take_dict_items(object: &mut Map<String, serde_json::Value>) -> Result<Vec<DictEntry>, String> {
+    match object.remove("items") {
+        Some(serde_json::Value::Array(items)) => items.into_iter().map(parse_dict_entry).collect(),
+        Some(_) => Err("Clew dict items field must be an array".to_string()),
+        None => Err("missing Clew dict items field".to_string()),
+    }
+}
+
+fn parse_dict_entry(value: serde_json::Value) -> Result<DictEntry, String> {
+    let serde_json::Value::Object(mut object) = value else {
+        return Err("Clew dict entry must be an object".to_string());
+    };
+    let key = object.remove("key").ok_or("missing Clew dict entry key")?;
+    let value = object
+        .remove("value")
+        .ok_or("missing Clew dict entry value")?;
+    Ok(DictEntry {
+        key: parse_value(key)?,
+        value: parse_value(value)?,
+    })
+}
+
+fn take_attributes(
+    object: &mut Map<String, serde_json::Value>,
+) -> Result<BTreeMap<String, Value>, String> {
+    match object.remove("attributes") {
+        Some(serde_json::Value::Object(attributes)) => attributes
+            .into_iter()
+            .map(|(key, value)| parse_value(value).map(|value| (key, value)))
+            .collect(),
+        Some(_) => Err("Clew object attributes field must be an object".to_string()),
+        None => Err("missing Clew object attributes field".to_string()),
+    }
+}
+
+fn take_string(object: &mut Map<String, serde_json::Value>, field: &str) -> Result<String, String> {
+    match object.remove(field) {
+        Some(serde_json::Value::String(value)) => Ok(value),
+        Some(_) => Err(format!("Clew {field} field must be a string")),
+        None => Err(format!("missing Clew {field} field")),
+    }
+}
+
+fn take_optional_string(
+    object: &mut Map<String, serde_json::Value>,
+    field: &str,
+) -> Result<Option<String>, String> {
+    match object.remove(field) {
+        Some(serde_json::Value::String(value)) => Ok(Some(value)),
+        Some(serde_json::Value::Null) | None => Ok(None),
+        Some(_) => Err(format!("Clew {field} field must be a string or null")),
+    }
+}
+
+fn take_optional_number(
+    object: &mut Map<String, serde_json::Value>,
+    field: &str,
+) -> Result<Option<Number>, String> {
+    match object.remove(field) {
+        Some(serde_json::Value::Number(value)) => Ok(Some(value)),
+        Some(serde_json::Value::Null) | None => Ok(None),
+        Some(_) => Err(format!("Clew {field} field must be a number or null")),
+    }
+}
+
+fn take_u64(object: &mut Map<String, serde_json::Value>, field: &str) -> Result<u64, String> {
+    match object.remove(field) {
+        Some(serde_json::Value::Number(value)) => value
+            .as_u64()
+            .ok_or_else(|| format!("Clew {field} field must be a non-negative integer")),
+        Some(_) => Err(format!("Clew {field} field must be a number")),
+        None => Err(format!("missing Clew {field} field")),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn document_with(roots: &str) -> String {
+        format!(
+            r#"{{
+                "schema":"clew-renpy-ast-v1",
+                "source":"/tmp/game/script.rpyc",
+                "game_root":"/tmp/game",
+                "slot":2,
+                "python_version":"2.7.18",
+                "renpy_version":"8.4.0",
+                "metadata":null,
+                "roots":{roots}
+            }}"#
+        )
+    }
+
+    #[test]
+    fn parses_minimal_document() {
+        let document = from_bytes(document_with("null").as_bytes()).unwrap();
+
+        assert_eq!(document.schema, "clew-renpy-ast-v1");
+        assert_eq!(document.slot, 2);
+        assert_eq!(document.roots, Value::Null);
+    }
+
+    #[test]
+    fn rejects_wrong_schema() {
+        let json = br#"{
+            "schema":"other",
+            "source":"/tmp/game/script.rpyc",
+            "game_root":"/tmp/game",
+            "slot":2,
+            "python_version":"2.7.18",
+            "renpy_version":null,
+            "metadata":null,
+            "roots":null
+        }"#;
+
+        assert!(matches!(
+            from_bytes(json).unwrap_err(),
+            Error::SchemaMismatch {
+                expected: "clew-renpy-ast-v1",
+                actual
+            } if actual == "other"
+        ));
+    }
+
+    #[test]
+    fn parses_primitive_values() {
+        let document = from_bytes(
+            document_with(r#"{"type":"list","items":[null,true,7,1.5,"text"]}"#).as_bytes(),
+        )
+        .unwrap();
+
+        assert_eq!(
+            document.roots,
+            Value::List(vec![
+                Value::Null,
+                Value::Bool(true),
+                Value::Number(Number::from(7)),
+                Value::Number(Number::from_f64(1.5).unwrap()),
+                Value::String("text".to_string()),
+            ])
+        );
+    }
+
+    #[test]
+    fn parses_tuple_and_dict_values() {
+        let document = from_bytes(
+            document_with(r#"{"type":"dict","items":[{"key":"names","value":{"type":"tuple","items":["a","b"]}}]}"#)
+                .as_bytes(),
+        )
+        .unwrap();
+
+        assert_eq!(
+            document.roots,
+            Value::Dict(vec![DictEntry {
+                key: Value::String("names".to_string()),
+                value: Value::Tuple(vec![
+                    Value::String("a".to_string()),
+                    Value::String("b".to_string())
+                ]),
+            }])
+        );
+    }
+
+    #[test]
+    fn parses_object_and_reference_values() {
+        let document = from_bytes(
+            document_with(
+                r#"{"type":"list","items":[{"type":"object","id":1,"class":"renpy.ast.Label","attributes":{"name":"start","next":{"type":"ref","id":1}}},{"type":"ref","id":1}]}"#,
+            )
+            .as_bytes(),
+        )
+        .unwrap();
+
+        let Value::List(items) = document.roots else {
+            panic!("roots should be a list");
+        };
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[1], Value::Ref { id: 1 });
+        let Value::Object(object) = &items[0] else {
+            panic!("first item should be an object");
+        };
+        assert_eq!(object.id, 1);
+        assert_eq!(object.class_name, "renpy.ast.Label");
+        assert_eq!(
+            object.attributes["name"],
+            Value::String("start".to_string())
+        );
+    }
+
+    #[test]
+    fn parses_special_tag_values() {
+        let document = from_bytes(
+            document_with(
+                r#"{"type":"list","items":[{"type":"function","name":"f","module":"m","filename":"file.py","firstlineno":9},{"type":"method","name":"g","module":"m","receiver_class":"C"},{"type":"python-type","name":"Label","module":"renpy.ast"},{"type":"repr","class":"C","repr":"<C>"},{"type":"bytes","repr":"b'abc'"},{"type":"truncated","class":"C","repr":"<C>"},{"type":"recursion","class":"list"},{"type":"attribute-error","class":"Exception","repr":"Exception('x')"}]}"#,
+            )
+            .as_bytes(),
+        )
+        .unwrap();
+
+        let Value::List(items) = document.roots else {
+            panic!("roots should be a list");
+        };
+        assert_eq!(items.len(), 8);
+        assert!(matches!(items[0], Value::Function(_)));
+        assert!(matches!(items[1], Value::Method(_)));
+        assert!(matches!(items[2], Value::PythonType(_)));
+        assert!(matches!(items[3], Value::Repr { .. }));
+        assert!(matches!(items[4], Value::Bytes { .. }));
+        assert!(matches!(items[5], Value::Truncated { .. }));
+        assert!(matches!(items[6], Value::Recursion { .. }));
+        assert!(matches!(items[7], Value::AttributeError { .. }));
+    }
+
+    #[test]
+    fn rejects_unknown_tags() {
+        let error = from_bytes(document_with(r#"{"type":"future"}"#).as_bytes()).unwrap_err();
+
+        match error {
+            Error::Json(error) => assert!(
+                error
+                    .to_string()
+                    .contains("unknown Clew value type: future")
+            ),
+            Error::SchemaMismatch { .. } => panic!("unknown tags should be JSON shape errors"),
+        }
+    }
+}

src/ariadne/lib.rs

@@ -4,6 +4,7 @@ use std::path::PathBuf;
 
 use crate::pipeline::Stage;
 
+pub mod clew;
 pub mod pipeline;
 
 const OUTPUT_GRAPH_DIR: &str = "graph";
@@ -31,7 +32,10 @@ fn get_output_dir(config: &Config, stage: Stage) -> Result<PathBuf> {
     if output_dir.exists() && !output_dir.is_dir() {
         return Err(io::Error::new(
             io::ErrorKind::NotADirectory,
-            format!("output path is not a directory: {path}", path = output_dir.display()),
+            format!(
+                "output path is not a directory: {path}",
+                path = output_dir.display()
+            ),
         ));
     }