build(rust): vendor rnix parser crate

third_party/crates/rnix-0.14.0/src/ast/expr_ext.rs

@@ -0,0 +1,29 @@
+use crate::ast::{self, support::*};
+
+// this is a separate type because it mixes tokens and nodes
+// for example, a Str is a node because it can contain nested subexpressions but an Integer is a token.
+// This means that we have to write it out manually instead of using the macro to create the type for us.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum LiteralKind {
+    Float(ast::Float),
+    Integer(ast::Integer),
+    Uri(ast::Uri),
+}
+
+impl ast::Literal {
+    pub fn kind(&self) -> LiteralKind {
+        if let Some(it) = token(self) {
+            return LiteralKind::Float(it);
+        }
+
+        if let Some(it) = token(self) {
+            return LiteralKind::Integer(it);
+        }
+
+        if let Some(it) = token(self) {
+            return LiteralKind::Uri(it);
+        }
+
+        unreachable!()
+    }
+}

third_party/crates/rnix-0.14.0/src/ast/interpol.rs

@@ -0,0 +1,5 @@
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum InterpolPart<T> {
+    Literal(T),
+    Interpolation(super::Interpol),
+}

third_party/crates/rnix-0.14.0/src/ast/nodes.rs

@@ -0,0 +1,464 @@
+use crate::{NixLanguage, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxToken};
+
+use super::{operators::BinOpKind, support::*, AstNode, UnaryOpKind};
+use rowan::ast::{AstChildren, AstNode as OtherAstNode};
+
+pub trait HasEntry: AstNode {
+    fn entries(&self) -> AstChildren<Entry>
+    where
+        Self: Sized,
+    {
+        children(self)
+    }
+
+    fn attrpath_values(&self) -> AstChildren<AttrpathValue>
+    where
+        Self: Sized,
+    {
+        children(self)
+    }
+
+    fn inherits(&self) -> AstChildren<Inherit>
+    where
+        Self: Sized,
+    {
+        children(self)
+    }
+}
+
+macro_rules! node {
+    (
+        #[from($kind:ident)]
+        $(#[$meta:meta])*
+        struct $name:ident;
+    ) => {
+        #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+        $(#[$meta])*
+        pub struct $name(pub(super) SyntaxNode);
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                std::fmt::Display::fmt(self.syntax(), f)
+            }
+        }
+
+        impl rowan::ast::AstNode for $name {
+            type Language = crate::NixLanguage;
+
+            fn can_cast(kind: crate::SyntaxKind) -> bool {
+                kind == $kind
+            }
+
+            fn cast(from: SyntaxNode) -> Option<Self> {
+                if Self::can_cast(from.kind()) {
+                    Some(Self(from))
+                } else {
+                    None
+                }
+            }
+            fn syntax(&self) -> &SyntaxNode {
+                &self.0
+            }
+        }
+
+        impl $name {
+            pub const KIND: SyntaxKind = $kind;
+        }
+    };
+    (
+        #[from($($variant:ident),* $(,)?)]
+        $(#[$meta:meta])*
+        enum $name:ident;
+    ) => {
+        #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+        $(#[$meta])*
+        pub enum $name {
+            $(
+                $variant($variant),
+            )*
+        }
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                std::fmt::Display::fmt(self.syntax(), f)
+            }
+        }
+
+        impl rowan::ast::AstNode for $name {
+            type Language = NixLanguage;
+
+            fn can_cast(kind: SyntaxKind) -> bool {
+                matches!(kind, $($variant::KIND)|*)
+            }
+
+            fn cast(syntax: SyntaxNode) -> Option<Self> {
+                let res = match syntax.kind() {
+                    $(
+                        $variant::KIND => $name::$variant($variant(syntax))
+                    ),*,
+                    _ => return None,
+                };
+                Some(res)
+            }
+
+            fn syntax(&self) -> &SyntaxNode {
+                match self {
+                    $(
+                        $name::$variant(it) => &it.0,
+                    )*
+                }
+            }
+        }
+
+        $(
+            impl From<$variant> for $name {
+                fn from(node: $variant) -> $name { $name::$variant(node) }
+            }
+
+            impl TryFrom<$name> for $variant {
+                type Error = ();
+
+                fn try_from(node: $name) -> Result<$variant, ()> {
+                    match node {
+                        $name::$variant(it) => Ok(it),
+                        _ => Err(()),
+                    }
+                }
+            }
+        )*
+    };
+}
+
+macro_rules! tg {
+    (
+        $(#[doc=$doc:tt])?
+        $name:ident,
+        $token:tt
+    ) => {
+        $(#[doc=$doc])?
+        pub fn $name(&self) -> Option<SyntaxToken> {
+            token_u(self, T![$token])
+        }
+    };
+}
+
+macro_rules! ng {
+    (
+        $(#[$meta:meta])*
+        $name:ident,
+        $ty:ty,
+        $i:expr
+    ) => {
+        $(#[$meta])*
+        pub fn $name(&self) -> Option<$ty> {
+            nth(self, $i)
+        }
+    };
+    (
+        $(#[$meta:meta])*
+        $name:ident,
+        [$ty:ty]
+    ) => {
+        $(#[$meta])*
+        pub fn $name(&self) -> AstChildren<$ty> {
+            children(self)
+        }
+    };
+}
+
+node! { #[from(NODE_LITERAL)] struct Literal; }
+
+node! {
+    #[from(
+        Apply,
+        Assert,
+        Error,
+        IfElse,
+        Select,
+        Str,
+        PathAbs,
+        PathRel,
+        PathHome,
+        PathSearch,
+        Literal,
+        Lambda,
+        LegacyLet,
+        LetIn,
+        List,
+        BinOp,
+        Paren,
+        Root,
+        AttrSet,
+        UnaryOp,
+        Ident,
+        With,
+        HasAttr,
+        CurPos,
+    )]
+    /// An expression. The fundamental nix ast type.
+    enum Expr;
+}
+
+node! {
+    #[from(
+        Ident,
+        Dynamic,
+        Str,
+    )]
+    enum Attr;
+}
+
+node! { #[from(NODE_IDENT)] struct Ident; }
+
+impl Ident {
+    // A NODE_IDENT may either have a nested TOKEN_OR if the identifier was "or" or TOKEN_IDENT for everything else
+    tg! { ident_token, TOKEN_IDENT }
+}
+
+node! { #[from(NODE_APPLY)] struct Apply; }
+
+impl Apply {
+    ng! { lambda, Expr, 0 }
+    ng! { argument, Expr, 1 }
+}
+
+node! { #[from(NODE_ASSERT)] struct Assert; }
+
+impl Assert {
+    tg! { assert_token, assert }
+    ng! { condition, Expr, 0 }
+    ng! { body, Expr, 1 }
+}
+
+node! { #[from(NODE_ATTRPATH)] struct Attrpath; }
+
+impl Attrpath {
+    ng! { attrs, [Attr] }
+}
+
+node! { #[from(NODE_DYNAMIC)] struct Dynamic; }
+
+impl Dynamic {
+    tg! { interpol_start_token, TOKEN_INTERPOL_START }
+    ng! { expr, Expr, 0 }
+    tg! { interpol_end_token, TOKEN_INTERPOL_END }
+}
+
+node! { #[from(NODE_ERROR)] struct Error; }
+
+node! { #[from(NODE_IF_ELSE)] struct IfElse; }
+
+impl IfElse {
+    tg! { if_token, if }
+    ng! { condition, Expr, 0 }
+    tg! { then_token, then }
+    ng! { body, Expr, 1 }
+    tg! { else_token, else }
+    ng! { else_body, Expr, 2 }
+}
+
+node! { #[from(NODE_SELECT)] struct Select; }
+
+impl Select {
+    ng! { expr, Expr, 0 }
+    tg! { dot_token, . }
+    ng! { attrpath, Attrpath, 0 }
+    tg! { or_token, or }
+    ng! { default_expr, Expr, 1 }
+}
+
+node! { #[from(NODE_INHERIT)] struct Inherit; }
+
+impl Inherit {
+    tg! { inherit_token, inherit }
+    ng! { from, InheritFrom, 0 }
+    ng! { attrs, [Attr] }
+}
+
+node! { #[from(NODE_INHERIT_FROM)] struct InheritFrom; }
+
+impl InheritFrom {
+    tg! { l_paren_token, '(' }
+    ng! { expr, Expr, 0 }
+    tg! { r_paren_token, ')' }
+}
+
+node! { #[from(NODE_PATH_ABS)]    struct PathAbs; }
+node! { #[from(NODE_PATH_REL)]    struct PathRel; }
+node! { #[from(NODE_PATH_HOME)]   struct PathHome; }
+node! { #[from(NODE_PATH_SEARCH)] struct PathSearch; }
+
+node! {
+    #[from(
+        PathAbs,
+        PathRel,
+        PathHome,
+        PathSearch,
+    )]
+    enum Path;
+}
+
+node! { #[from(NODE_STRING)] struct Str; }
+
+node! { #[from(NODE_INTERPOL)] struct Interpol; }
+
+impl Interpol {
+    ng! { expr, Expr, 0 }
+}
+
+node! { #[from(NODE_LAMBDA)] struct Lambda; }
+
+impl Lambda {
+    ng! { param, Param, 0 }
+    tg! { token_colon, : }
+    ng! { body, Expr, 0 }
+}
+
+node! { #[from(NODE_LEGACY_LET)] struct LegacyLet; }
+
+impl HasEntry for LegacyLet {}
+
+impl LegacyLet {
+    tg! { let_token, let }
+    tg! { curly_open_token, '{' }
+    tg! { curly_close_token, '}' }
+}
+
+node! { #[from(NODE_LET_IN)] struct LetIn; }
+
+impl HasEntry for LetIn {}
+
+impl LetIn {
+    tg! { let_token, let }
+    tg! { in_token, in }
+    ng! { body, Expr, 0 }
+}
+
+node! { #[from(NODE_LIST)] struct List; }
+
+impl List {
+    tg! { l_brack_token, '[' }
+    ng! { items, [Expr] }
+    tg! { r_brack_token, ']' }
+}
+
+node! { #[from(NODE_BIN_OP)] struct BinOp; }
+
+impl BinOp {
+    ng! { lhs, Expr, 0 }
+
+    pub fn operator(&self) -> Option<BinOpKind> {
+        children_tokens_u(self).find_map(|t| BinOpKind::from_kind(t.kind()))
+    }
+
+    ng! { rhs, Expr, 1 }
+}
+
+node! { #[from(NODE_PAREN)] struct Paren; }
+
+impl Paren {
+    tg! { l_paren_token, '(' }
+    ng! { expr, Expr, 0 }
+    tg! { r_paren_token, ')' }
+}
+
+node! { #[from(NODE_PAT_BIND)] struct PatBind; }
+
+impl PatBind {
+    ng! { ident, Ident, 0 }
+}
+
+node! { #[from(NODE_PAT_ENTRY)] struct PatEntry; }
+
+impl PatEntry {
+    ng! { ident, Ident, 0 }
+    tg! { question_token, ? }
+    ng! { default, Expr, 1 }
+}
+
+node! { #[from(NODE_IDENT_PARAM)] struct IdentParam; }
+
+impl IdentParam {
+    ng! { ident, Ident, 0 }
+}
+
+node! {
+    #[from(
+        Pattern,
+        IdentParam,
+    )]
+    enum Param;
+}
+
+node! { #[from(NODE_PATTERN)] struct Pattern; }
+
+impl Pattern {
+    tg! { at_token, @ }
+    ng! { pat_entries, [PatEntry] }
+    tg! { ellipsis_token, ... }
+    ng! { pat_bind, PatBind, 0 }
+}
+
+node! { #[from(NODE_ROOT)] struct Root; }
+
+impl Root {
+    ng! { expr, Expr, 0 }
+}
+
+node! { #[from(NODE_ATTR_SET)] struct AttrSet; }
+
+impl HasEntry for AttrSet {}
+
+impl AttrSet {
+    tg! { rec_token, rec }
+    tg! { l_curly_token, '{' }
+    tg! { r_curly_token, '}' }
+}
+
+node! {
+    #[from(
+        Inherit,
+        AttrpathValue,
+    )]
+    enum Entry;
+}
+
+node! { #[from(NODE_ATTRPATH_VALUE)] struct AttrpathValue; }
+
+impl AttrpathValue {
+    ng! { attrpath, Attrpath, 0 }
+    tg! { assign_token, = }
+    ng! { value, Expr, 0 }
+}
+
+node! { #[from(NODE_UNARY_OP)] struct UnaryOp; }
+
+impl UnaryOp {
+    pub fn operator(&self) -> Option<UnaryOpKind> {
+        children_tokens_u(self).find_map(|t| UnaryOpKind::from_kind(t.kind()))
+    }
+    ng! { expr, Expr, 0 }
+}
+
+node! { #[from(NODE_WITH)] struct With; }
+
+impl With {
+    tg! { with_token, with }
+    ng! { namespace, Expr, 0 }
+    tg! { semicolon_token, ; }
+    ng! { body, Expr, 1 }
+}
+
+node! { #[from(NODE_HAS_ATTR)] struct HasAttr; }
+
+impl HasAttr {
+    ng! { expr, Expr, 0 }
+    tg! { question_token, ? }
+    ng! { attrpath, Attrpath, 0 }
+}
+
+node! { #[from(NODE_CUR_POS)] struct CurPos; }
+
+impl CurPos {
+    tg! { cur_pos_token, __curPos }
+}

third_party/crates/rnix-0.14.0/src/ast/operators.rs

@@ -0,0 +1,71 @@
+use crate::SyntaxKind::{self, *};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum BinOpKind {
+    Concat,
+    Update,
+
+    Add,
+    Sub,
+    Mul,
+    Div,
+
+    And,
+    Equal,
+    Implication,
+    Less,
+    LessOrEq,
+    More,
+    MoreOrEq,
+    NotEqual,
+    Or,
+    PipeRight,
+    PipeLeft,
+}
+
+impl BinOpKind {
+    /// Get the operation kind from a SyntaxKind in the AST
+    pub fn from_kind(token: SyntaxKind) -> Option<Self> {
+        match token {
+            TOKEN_CONCAT => Some(BinOpKind::Concat),
+            TOKEN_UPDATE => Some(BinOpKind::Update),
+
+            TOKEN_ADD => Some(BinOpKind::Add),
+            TOKEN_SUB => Some(BinOpKind::Sub),
+            TOKEN_MUL => Some(BinOpKind::Mul),
+            TOKEN_DIV => Some(BinOpKind::Div),
+
+            TOKEN_AND_AND => Some(BinOpKind::And),
+            TOKEN_EQUAL => Some(BinOpKind::Equal),
+            TOKEN_IMPLICATION => Some(BinOpKind::Implication),
+            TOKEN_LESS => Some(BinOpKind::Less),
+            TOKEN_LESS_OR_EQ => Some(BinOpKind::LessOrEq),
+            TOKEN_MORE => Some(BinOpKind::More),
+            TOKEN_MORE_OR_EQ => Some(BinOpKind::MoreOrEq),
+            TOKEN_NOT_EQUAL => Some(BinOpKind::NotEqual),
+            TOKEN_OR_OR => Some(BinOpKind::Or),
+
+            TOKEN_PIPE_RIGHT => Some(BinOpKind::PipeRight),
+            TOKEN_PIPE_LEFT => Some(BinOpKind::PipeLeft),
+
+            _ => None,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum UnaryOpKind {
+    Invert,
+    Negate,
+}
+
+impl UnaryOpKind {
+    /// Get the operation kind from a token in the AST
+    pub fn from_kind(kind: SyntaxKind) -> Option<Self> {
+        match kind {
+            TOKEN_INVERT => Some(UnaryOpKind::Invert),
+            TOKEN_SUB => Some(UnaryOpKind::Negate),
+            _ => None,
+        }
+    }
+}

third_party/crates/rnix-0.14.0/src/ast/path_util.rs

@@ -0,0 +1,204 @@
+use crate::kinds::SyntaxKind::{TOKEN_PATH_ABS, TOKEN_PATH_HOME, TOKEN_PATH_REL};
+
+use rowan::{ast::AstNode as OtherAstNode, NodeOrToken};
+
+pub use super::nodes::Path;
+use super::{
+    nodes::{PathAbs, PathHome, PathRel, PathSearch},
+    AstToken, InterpolPart, PathContent,
+};
+use crate::ast;
+
+fn extract_path_parts<T: ast::AstNode>(node: &T) -> Vec<InterpolPart<PathContent>> {
+    node.syntax()
+        .children_with_tokens()
+        .map(|child| match child {
+            NodeOrToken::Token(token) => {
+                debug_assert!(matches!(
+                    token.kind(),
+                    TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME
+                ));
+                InterpolPart::Literal(PathContent::cast(token).unwrap())
+            }
+            NodeOrToken::Node(node) => {
+                InterpolPart::Interpolation(ast::Interpol::cast(node).unwrap())
+            }
+        })
+        .collect()
+}
+
+// Direct methods for interpolatable path types
+impl PathAbs {
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        extract_path_parts(self)
+    }
+}
+
+impl PathRel {
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        extract_path_parts(self)
+    }
+}
+
+impl PathHome {
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        extract_path_parts(self)
+    }
+}
+
+// Direct methods for search path
+impl PathSearch {
+    /// Get the content of a search path
+    pub fn content(&self) -> Option<PathContent> {
+        self.syntax()
+            .children_with_tokens()
+            .filter_map(|child| child.into_token().and_then(PathContent::cast))
+            .next()
+    }
+}
+
+/// Extension methods for the Path enum
+impl Path {
+    /// Get parts from any path type in a unified way
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        match self {
+            // For interpolatable paths, get their parts
+            Path::PathAbs(p) => p.parts(),
+            Path::PathRel(p) => p.parts(),
+            Path::PathHome(p) => p.parts(),
+            // For search paths, return a single literal component if content exists
+            Path::PathSearch(p) => {
+                if let Some(content) = p.content() {
+                    vec![InterpolPart::Literal(content)]
+                } else {
+                    vec![]
+                }
+            }
+        }
+    }
+
+    pub fn is_search(&self) -> bool {
+        matches!(self, Path::PathSearch(_))
+    }
+
+    pub fn is_interpolatable(&self) -> bool {
+        !self.is_search()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rowan::ast::AstNode;
+
+    use super::InterpolPart;
+    use crate::{
+        ast::{self, Path},
+        Root,
+    };
+
+    #[test]
+    fn test_path_types() {
+        // Absolute path
+        let inp = "/foo/bar";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathAbs(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+            assert!(path.is_interpolatable());
+            assert!(!path.is_search());
+        }
+
+        // Search path
+        let inp = "<nixpkgs>";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathSearch(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+            assert!(!path.is_interpolatable());
+            assert!(path.is_search());
+        }
+    }
+
+    #[test]
+    fn test_parts() {
+        // Test parts with absolute path
+        let inp = "/foo/bar";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathAbs(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+
+            let parts = path.parts();
+            assert_eq!(parts.len(), 1);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "/foo/bar");
+                }
+                _ => panic!("Expected literal part"),
+            }
+        }
+
+        // Test parts with interpolated path
+        let inp = r#"./a/${"hello"}"#;
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathRel(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+
+            let parts = path.parts();
+            assert_eq!(parts.len(), 2);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "./a/");
+                }
+                _ => panic!("Expected literal part"),
+            }
+
+            match &parts[1] {
+                InterpolPart::Interpolation(_) => {} // Success
+                _ => panic!("Expected interpolation part"),
+            }
+        }
+
+        // Test parts with search path
+        let inp = "<nixpkgs>";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathSearch(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+
+            let parts = path.parts();
+            assert_eq!(parts.len(), 1);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "<nixpkgs>");
+                }
+                _ => panic!("Expected literal part"),
+            }
+        }
+    }
+
+    #[test]
+    fn direct_method_usage() {
+        // Test direct parts() method on PathAbs
+        let inp = "/foo/bar";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathAbs(p) = expr {
+            let parts = p.parts();
+            assert_eq!(parts.len(), 1);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "/foo/bar");
+                }
+                _ => panic!("Expected literal part"),
+            }
+        }
+
+        // Test direct content() method on PathSearch
+        let inp = "<nixpkgs>";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathSearch(p) = expr {
+            let content = p.content().expect("Expected content");
+            assert_eq!(content.text(), "<nixpkgs>");
+        }
+    }
+}

third_party/crates/rnix-0.14.0/src/ast/str_util.rs

@@ -0,0 +1,363 @@
+use crate::kinds::SyntaxKind::*;
+use rowan::{ast::AstNode as OtherAstNode, NodeOrToken};
+
+use crate::ast;
+
+use super::{support::children_tokens_u, AstToken, InterpolPart, StrContent};
+
+impl ast::Str {
+    pub fn parts(&self) -> impl Iterator<Item = InterpolPart<StrContent>> {
+        self.syntax().children_with_tokens().filter_map(|child| match child {
+            NodeOrToken::Token(token) if token.kind() == TOKEN_STRING_CONTENT => {
+                Some(InterpolPart::Literal(StrContent::cast(token).unwrap()))
+            }
+            NodeOrToken::Token(token) => {
+                assert!(token.kind() == TOKEN_STRING_START || token.kind() == TOKEN_STRING_END);
+                None
+            }
+            NodeOrToken::Node(node) => {
+                assert_eq!(node.kind(), NODE_INTERPOL);
+                Some(InterpolPart::Interpolation(ast::Interpol::cast(node).unwrap()))
+            }
+        })
+    }
+
+    pub fn normalized_parts(&self) -> Vec<InterpolPart<String>> {
+        let multiline = children_tokens_u(self).next().is_some_and(|t| t.text() == "''");
+        let mut is_first_literal = true;
+        let mut at_start_of_line = true;
+        let mut min_indent = 1000000;
+        let mut cur_indent = 0;
+        let mut n = 0;
+        let mut first_is_literal = false;
+
+        let parts: Vec<InterpolPart<StrContent>> = self.parts().collect();
+
+        if multiline {
+            for part in &parts {
+                match part {
+                    InterpolPart::Interpolation(_) => {
+                        if at_start_of_line {
+                            at_start_of_line = false;
+                            min_indent = min_indent.min(cur_indent);
+                        }
+                        n += 1;
+                    }
+                    InterpolPart::Literal(literal) => {
+                        let mut token_text = literal.syntax().text();
+
+                        if n == 0 {
+                            first_is_literal = true;
+                        }
+
+                        if is_first_literal && first_is_literal {
+                            is_first_literal = false;
+                            if let Some(p) = token_text.find('\n') {
+                                if token_text[0..p].chars().all(|c| c == ' ') {
+                                    token_text = &token_text[p + 1..]
+                                }
+                            }
+                        }
+
+                        for c in token_text.chars() {
+                            if at_start_of_line {
+                                if c == ' ' {
+                                    cur_indent += 1;
+                                } else if c == '\n' {
+                                    cur_indent = 0;
+                                } else {
+                                    at_start_of_line = false;
+                                    min_indent = min_indent.min(cur_indent);
+                                }
+                            } else if c == '\n' {
+                                at_start_of_line = true;
+                                cur_indent = 0;
+                            }
+                        }
+
+                        n += 1;
+                    }
+                }
+            }
+        }
+
+        let mut normalized_parts = Vec::new();
+        let mut cur_dropped = 0;
+        let mut i = 0;
+        is_first_literal = true;
+        at_start_of_line = true;
+
+        for part in parts {
+            match part {
+                InterpolPart::Interpolation(interpol) => {
+                    at_start_of_line = false;
+                    cur_dropped = 0;
+                    normalized_parts.push(InterpolPart::Interpolation(interpol));
+                    i += 1;
+                }
+                InterpolPart::Literal(literal) => {
+                    let mut token_text = literal.syntax().text();
+
+                    if multiline {
+                        if is_first_literal && first_is_literal {
+                            is_first_literal = false;
+                            if let Some(p) = token_text.find('\n') {
+                                if token_text[0..p].chars().all(|c| c == ' ') {
+                                    token_text = &token_text[p + 1..];
+                                    if token_text.is_empty() {
+                                        i += 1;
+                                        continue;
+                                    }
+                                }
+                            }
+                        }
+
+                        let mut str = String::new();
+                        for c in token_text.chars() {
+                            if at_start_of_line {
+                                if c == ' ' {
+                                    if cur_dropped >= min_indent {
+                                        str.push(c);
+                                    }
+                                    cur_dropped += 1;
+                                } else if c == '\n' {
+                                    cur_dropped = 0;
+                                    str.push(c);
+                                } else {
+                                    at_start_of_line = false;
+                                    cur_dropped = 0;
+                                    str.push(c);
+                                }
+                            } else {
+                                str.push(c);
+                                if c == '\n' {
+                                    at_start_of_line = true;
+                                }
+                            }
+                        }
+
+                        if i == n - 1 {
+                            if let Some(p) = str.rfind('\n') {
+                                if str[p + 1..].chars().all(|c| c == ' ') {
+                                    str.truncate(p + 1);
+                                }
+                            }
+                        }
+
+                        normalized_parts.push(InterpolPart::Literal(unescape(&str, multiline)));
+                        i += 1;
+                    } else {
+                        normalized_parts
+                            .push(InterpolPart::Literal(unescape(token_text, multiline)));
+                    }
+                }
+            }
+        }
+
+        normalized_parts
+    }
+}
+
+/// Interpret escape sequences in the nix string and return the converted value
+pub fn unescape(input: &str, multiline: bool) -> String {
+    let mut output = String::new();
+    let mut input = input.chars().peekable();
+    loop {
+        match input.next() {
+            None => break,
+            Some('"') if !multiline => break,
+            Some('\\') if !multiline => match input.next() {
+                None => break,
+                Some('n') => output.push('\n'),
+                Some('r') => output.push('\r'),
+                Some('t') => output.push('\t'),
+                Some(c) => output.push(c),
+            },
+            Some('\'') if multiline => match input.next() {
+                None => {
+                    output.push('\'');
+                }
+                Some('\'') => match input.peek() {
+                    Some('\'') => {
+                        input.next().unwrap();
+                        output.push_str("''");
+                    }
+                    Some('$') => {
+                        input.next().unwrap();
+                        output.push('$');
+                    }
+                    Some('\\') => {
+                        input.next().unwrap();
+                        match input.next() {
+                            None => break,
+                            Some('n') => output.push('\n'),
+                            Some('r') => output.push('\r'),
+                            Some('t') => output.push('\t'),
+                            Some(c) => output.push(c),
+                        }
+                    }
+                    _ => break,
+                },
+                Some(c) => {
+                    output.push('\'');
+                    output.push(c);
+                }
+            },
+            Some(c) => output.push(c),
+        }
+    }
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::Root;
+
+    use super::*;
+
+    #[test]
+    fn string_unescapes() {
+        assert_eq!(unescape(r#"Hello\n\"World\" :D"#, false), "Hello\n\"World\" :D");
+        assert_eq!(unescape(r#"\"Hello\""#, false), "\"Hello\"");
+
+        assert_eq!(unescape(r#"Hello''\n'''World''' :D"#, true), "Hello\n''World'' :D");
+        assert_eq!(unescape(r#""Hello""#, true), "\"Hello\"");
+    }
+    #[test]
+    fn parts_leading_ws() {
+        let inp = "''\n  hello\n  world''";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(str) => {
+                assert_eq!(
+                    str.normalized_parts(),
+                    vec![InterpolPart::Literal("hello\nworld".to_string())]
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+    #[test]
+    fn parts_trailing_ws_single_line() {
+        let inp = "''hello ''";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(str) => {
+                assert_eq!(
+                    str.normalized_parts(),
+                    vec![InterpolPart::Literal("hello ".to_string())]
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+    #[test]
+    fn parts_trailing_ws_multiline() {
+        let inp = "''hello\n ''";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(str) => {
+                assert_eq!(
+                    str.normalized_parts(),
+                    vec![InterpolPart::Literal("hello\n".to_string())]
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+    #[test]
+    fn parts() {
+        use crate::{NixLanguage, SyntaxNode};
+        use rowan::{GreenNodeBuilder, Language};
+
+        fn string_node(content: &str) -> ast::Str {
+            let mut builder = GreenNodeBuilder::new();
+            builder.start_node(NixLanguage::kind_to_raw(NODE_STRING));
+            builder.token(NixLanguage::kind_to_raw(TOKEN_STRING_START), "''");
+            builder.token(NixLanguage::kind_to_raw(TOKEN_STRING_CONTENT), content);
+            builder.token(NixLanguage::kind_to_raw(TOKEN_STRING_END), "''");
+            builder.finish_node();
+
+            ast::Str::cast(SyntaxNode::new_root(builder.finish())).unwrap()
+        }
+
+        let txtin = r#"
+                        |trailing-whitespace
+                              |trailing-whitespace
+                    This is a multiline string :D
+                      indented by two
+                    \'\'\'\'\
+                    ''${ interpolation was escaped }
+                    two single quotes: '''
+                    three single quotes: ''''
+                "#
+        .replace("|trailing-whitespace", "");
+
+        if let [InterpolPart::Literal(lit)] =
+            &ast::Str::normalized_parts(&string_node(txtin.as_str()))[..]
+        {
+            assert_eq!(lit,
+                // Get the below with nix repl
+                "    \n          \nThis is a multiline string :D\n  indented by two\n\\'\\'\\'\\'\\\n${ interpolation was escaped }\ntwo single quotes: ''\nthree single quotes: '''\n"
+            );
+        } else {
+            unreachable!();
+        }
+    }
+
+    #[test]
+    fn parts_ast() {
+        fn assert_eq_ast_ctn(it: &mut dyn Iterator<Item = InterpolPart<String>>, x: &str) {
+            let tmp = it.next().expect("unexpected EOF");
+            if let InterpolPart::Interpolation(astn) = tmp {
+                assert_eq!(astn.expr().unwrap().syntax().to_string(), x);
+            } else {
+                unreachable!("unexpected literal {:?}", tmp);
+            }
+        }
+
+        let inp = r#"''
+
+        This version of Nixpkgs requires Nix >= ${requiredVersion}, please upgrade:
+
+        - If you are running NixOS, `nixos-rebuild' can be used to upgrade your system.
+
+        - Alternatively, with Nix > 2.0 `nix upgrade-nix' can be used to imperatively
+          upgrade Nix. You may use `nix-env --version' to check which version you have.
+
+        - If you installed Nix using the install script (https://nixos.org/nix/install),
+          it is safe to upgrade by running it again:
+
+              curl -L https://nixos.org/nix/install | sh
+
+        For more information, please see the NixOS release notes at
+        https://nixos.org/nixos/manual or locally at
+        ${toString ./nixos/doc/manual/release-notes}.
+
+        If you need further help, see https://nixos.org/nixos/support.html
+      ''"#;
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(s) => {
+                let mut it = s.normalized_parts().into_iter();
+                assert_eq!(
+                    it.next().unwrap(),
+                    InterpolPart::Literal("\nThis version of Nixpkgs requires Nix >= ".to_string())
+                );
+                assert_eq_ast_ctn(&mut it, "requiredVersion");
+                assert_eq!(it.next().unwrap(), InterpolPart::Literal(
+                        ", please upgrade:\n\n- If you are running NixOS, `nixos-rebuild' can be used to upgrade your system.\n\n- Alternatively, with Nix > 2.0 `nix upgrade-nix' can be used to imperatively\n  upgrade Nix. You may use `nix-env --version' to check which version you have.\n\n- If you installed Nix using the install script (https://nixos.org/nix/install),\n  it is safe to upgrade by running it again:\n\n      curl -L https://nixos.org/nix/install | sh\n\nFor more information, please see the NixOS release notes at\nhttps://nixos.org/nixos/manual or locally at\n".to_string()
+                    ));
+                assert_eq_ast_ctn(&mut it, "toString ./nixos/doc/manual/release-notes");
+                assert_eq!(
+                    it.next().unwrap(),
+                    InterpolPart::Literal(
+                        ".\n\nIf you need further help, see https://nixos.org/nixos/support.html\n"
+                            .to_string()
+                    )
+                );
+            }
+            _ => unreachable!(),
+        }
+    }
+}

third_party/crates/rnix-0.14.0/src/ast/tokens.rs

@@ -0,0 +1,174 @@
+use core::num;
+
+use crate::{
+    ast::AstToken,
+    SyntaxKind::{self, *},
+    SyntaxToken,
+};
+
+macro_rules! token {
+    (
+        #[from($kind:ident)]
+        $(#[$meta:meta])*
+        struct $name:ident;
+    ) => {
+        #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+        $(#[$meta])*
+        pub struct $name(pub(super) SyntaxToken);
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                std::fmt::Display::fmt(self.syntax(), f)
+            }
+        }
+
+        impl AstToken for $name {
+            fn can_cast(kind: SyntaxKind) -> bool {
+                $kind == kind
+            }
+
+            fn cast(from: SyntaxToken) -> Option<Self> {
+                if from.kind() == $kind {
+                    Some(Self(from))
+                } else {
+                    None
+                }
+            }
+
+            fn syntax(&self) -> &SyntaxToken {
+                &self.0
+            }
+        }
+    };
+}
+
+token! { #[from(TOKEN_WHITESPACE)] struct Whitespace; }
+
+token! { #[from(TOKEN_COMMENT)] struct Comment; }
+
+impl Comment {
+    pub fn text(&self) -> &str {
+        let text = self.syntax().text();
+        // Handle both "#..." and "/*...*/" comments.
+        match text.strip_prefix('#') {
+            Some(s) => s,
+            None => text.strip_prefix(r#"/*"#).unwrap().strip_suffix(r#"*/"#).unwrap(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::ast;
+    use crate::match_ast;
+    use crate::Root;
+
+    use super::*;
+    use rowan::ast::AstNode;
+
+    #[test]
+    fn comment() {
+        let s = "# comment bruh
+/* this is a multiline comment wow
+asdfasdf
+asdfasdf */
+1 + 1
+/* last one */
+";
+        let comments: Vec<String> = Root::parse(s)
+            .ok()
+            .unwrap()
+            .syntax()
+            .children_with_tokens()
+            .filter_map(|e| match e {
+                rowan::NodeOrToken::Token(token) => match_ast! { match token {
+                    ast::Comment(it) => Some(it.text().to_string()),
+                    _ => None,
+                }},
+                rowan::NodeOrToken::Node(_) => None,
+            })
+            .collect();
+        let expected = vec![
+            " comment bruh",
+            " this is a multiline comment wow\nasdfasdf\nasdfasdf ",
+            " last one ",
+        ];
+
+        assert_eq!(comments, expected);
+    }
+}
+
+token! { #[from(TOKEN_FLOAT)] struct Float; }
+
+impl Float {
+    pub fn value(&self) -> Result<f64, num::ParseFloatError> {
+        self.syntax().text().parse()
+    }
+}
+
+token! { #[from(TOKEN_INTEGER)] struct Integer; }
+
+impl Integer {
+    pub fn value(&self) -> Result<i64, num::ParseIntError> {
+        self.syntax().text().parse()
+    }
+}
+
+// A literal part of a path (absolute / relative / home / search) without
+// distinguishing which concrete flavour it was taken from.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PathContent(pub(super) SyntaxToken);
+
+impl PathContent {
+    /// Returns the path as a string
+    pub fn text(&self) -> &str {
+        self.syntax().text()
+    }
+
+    /// Returns the kind of path token
+    pub fn path_kind(&self) -> SyntaxKind {
+        self.syntax().kind()
+    }
+
+    /// Returns true if this is an absolute path
+    pub fn is_absolute(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_ABS
+    }
+
+    /// Returns true if this is a relative path
+    pub fn is_relative(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_REL
+    }
+
+    /// Returns true if this is a home-relative path
+    pub fn is_home(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_HOME
+    }
+
+    /// Returns true if this is a search path
+    pub fn is_search(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_SEARCH
+    }
+}
+
+impl AstToken for PathContent {
+    fn can_cast(kind: SyntaxKind) -> bool {
+        matches!(kind, TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME | TOKEN_PATH_SEARCH)
+    }
+
+    fn cast(from: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(from.kind()) {
+            Some(Self(from))
+        } else {
+            None
+        }
+    }
+
+    fn syntax(&self) -> &SyntaxToken {
+        &self.0
+    }
+}
+
+token! { #[from(TOKEN_STRING_CONTENT)] struct StrContent; }
+
+token! { #[from(TOKEN_URI)] struct Uri; }