build(rust): vendor rnix parser crate

2026-09-16 15:51:43 -04:00
1 parent cbdf482422
commit 82ad65a3a4
@@ -0,0 +1,73 @@
+//! Provides a type system for the AST, in some sense
+
+mod expr_ext;
+mod interpol;
+mod nodes;
+mod operators;
+mod path_util;
+
+pub use path_util::Path;
+mod str_util;
+mod tokens;
+
+use crate::{NixLanguage, SyntaxKind, SyntaxToken};
+
+pub use expr_ext::LiteralKind;
+pub use interpol::*;
+pub use nodes::*;
+pub use operators::{BinOpKind, UnaryOpKind};
+pub use tokens::*;
+
+pub trait AstNode: rowan::ast::AstNode<Language = NixLanguage> {}
+
+impl<T> AstNode for T where T: rowan::ast::AstNode<Language = NixLanguage> {}
+
+pub trait AstToken {
+    fn can_cast(from: SyntaxKind) -> bool
+    where
+        Self: Sized;
+
+    /// Cast an untyped token into this strongly-typed token. This will return
+    /// None if the type was not correct.
+    fn cast(from: SyntaxToken) -> Option<Self>
+    where
+        Self: Sized;
+
+    fn syntax(&self) -> &SyntaxToken;
+}
+
+mod support {
+    use rowan::ast::AstChildren;
+
+    use super::{AstNode, AstToken};
+    use crate::{SyntaxElement, SyntaxKind, SyntaxToken};
+
+    pub(super) fn nth<N: AstNode, NN: AstNode>(parent: &N, n: usize) -> Option<NN> {
+        parent.syntax().children().flat_map(NN::cast).nth(n)
+    }
+
+    pub(super) fn children<N: AstNode, NN: AstNode>(parent: &N) -> AstChildren<NN> {
+        rowan::ast::support::children(parent.syntax())
+    }
+
+    pub(super) fn token<N: AstNode, T: AstToken>(parent: &N) -> Option<T> {
+        children_tokens(parent).next()
+    }
+
+    /// Token untyped
+    pub(super) fn token_u<N: AstNode>(parent: &N, kind: SyntaxKind) -> Option<SyntaxToken> {
+        children_tokens_u(parent).find(|it| it.kind() == kind)
+    }
+
+    pub(super) fn children_tokens<N: AstNode, T: AstToken>(parent: &N) -> impl Iterator<Item = T> {
+        parent
+            .syntax()
+            .children_with_tokens()
+            .filter_map(SyntaxElement::into_token)
+            .filter_map(T::cast)
+    }
+
+    pub(super) fn children_tokens_u<N: AstNode>(parent: &N) -> impl Iterator<Item = SyntaxToken> {
+        parent.syntax().children_with_tokens().filter_map(SyntaxElement::into_token)
+    }
+}
@@ -0,0 +1,29 @@
+use crate::ast::{self, support::*};
+
+// this is a separate type because it mixes tokens and nodes
+// for example, a Str is a node because it can contain nested subexpressions but an Integer is a token.
+// This means that we have to write it out manually instead of using the macro to create the type for us.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum LiteralKind {
+    Float(ast::Float),
+    Integer(ast::Integer),
+    Uri(ast::Uri),
+}
+
+impl ast::Literal {
+    pub fn kind(&self) -> LiteralKind {
+        if let Some(it) = token(self) {
+            return LiteralKind::Float(it);
+        }
+
+        if let Some(it) = token(self) {
+            return LiteralKind::Integer(it);
+        }
+
+        if let Some(it) = token(self) {
+            return LiteralKind::Uri(it);
+        }
+
+        unreachable!()
+    }
+}
@@ -0,0 +1,5 @@
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum InterpolPart<T> {
+    Literal(T),
+    Interpolation(super::Interpol),
+}
@@ -0,0 +1,464 @@
+use crate::{NixLanguage, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxToken};
+
+use super::{operators::BinOpKind, support::*, AstNode, UnaryOpKind};
+use rowan::ast::{AstChildren, AstNode as OtherAstNode};
+
+pub trait HasEntry: AstNode {
+    fn entries(&self) -> AstChildren<Entry>
+    where
+        Self: Sized,
+    {
+        children(self)
+    }
+
+    fn attrpath_values(&self) -> AstChildren<AttrpathValue>
+    where
+        Self: Sized,
+    {
+        children(self)
+    }
+
+    fn inherits(&self) -> AstChildren<Inherit>
+    where
+        Self: Sized,
+    {
+        children(self)
+    }
+}
+
+macro_rules! node {
+    (
+        #[from($kind:ident)]
+        $(#[$meta:meta])*
+        struct $name:ident;
+    ) => {
+        #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+        $(#[$meta])*
+        pub struct $name(pub(super) SyntaxNode);
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                std::fmt::Display::fmt(self.syntax(), f)
+            }
+        }
+
+        impl rowan::ast::AstNode for $name {
+            type Language = crate::NixLanguage;
+
+            fn can_cast(kind: crate::SyntaxKind) -> bool {
+                kind == $kind
+            }
+
+            fn cast(from: SyntaxNode) -> Option<Self> {
+                if Self::can_cast(from.kind()) {
+                    Some(Self(from))
+                } else {
+                    None
+                }
+            }
+            fn syntax(&self) -> &SyntaxNode {
+                &self.0
+            }
+        }
+
+        impl $name {
+            pub const KIND: SyntaxKind = $kind;
+        }
+    };
+    (
+        #[from($($variant:ident),* $(,)?)]
+        $(#[$meta:meta])*
+        enum $name:ident;
+    ) => {
+        #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+        $(#[$meta])*
+        pub enum $name {
+            $(
+                $variant($variant),
+            )*
+        }
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                std::fmt::Display::fmt(self.syntax(), f)
+            }
+        }
+
+        impl rowan::ast::AstNode for $name {
+            type Language = NixLanguage;
+
+            fn can_cast(kind: SyntaxKind) -> bool {
+                matches!(kind, $($variant::KIND)|*)
+            }
+
+            fn cast(syntax: SyntaxNode) -> Option<Self> {
+                let res = match syntax.kind() {
+                    $(
+                        $variant::KIND => $name::$variant($variant(syntax))
+                    ),*,
+                    _ => return None,
+                };
+                Some(res)
+            }
+
+            fn syntax(&self) -> &SyntaxNode {
+                match self {
+                    $(
+                        $name::$variant(it) => &it.0,
+                    )*
+                }
+            }
+        }
+
+        $(
+            impl From<$variant> for $name {
+                fn from(node: $variant) -> $name { $name::$variant(node) }
+            }
+
+            impl TryFrom<$name> for $variant {
+                type Error = ();
+
+                fn try_from(node: $name) -> Result<$variant, ()> {
+                    match node {
+                        $name::$variant(it) => Ok(it),
+                        _ => Err(()),
+                    }
+                }
+            }
+        )*
+    };
+}
+
+macro_rules! tg {
+    (
+        $(#[doc=$doc:tt])?
+        $name:ident,
+        $token:tt
+    ) => {
+        $(#[doc=$doc])?
+        pub fn $name(&self) -> Option<SyntaxToken> {
+            token_u(self, T![$token])
+        }
+    };
+}
+
+macro_rules! ng {
+    (
+        $(#[$meta:meta])*
+        $name:ident,
+        $ty:ty,
+        $i:expr
+    ) => {
+        $(#[$meta])*
+        pub fn $name(&self) -> Option<$ty> {
+            nth(self, $i)
+        }
+    };
+    (
+        $(#[$meta:meta])*
+        $name:ident,
+        [$ty:ty]
+    ) => {
+        $(#[$meta])*
+        pub fn $name(&self) -> AstChildren<$ty> {
+            children(self)
+        }
+    };
+}
+
+node! { #[from(NODE_LITERAL)] struct Literal; }
+
+node! {
+    #[from(
+        Apply,
+        Assert,
+        Error,
+        IfElse,
+        Select,
+        Str,
+        PathAbs,
+        PathRel,
+        PathHome,
+        PathSearch,
+        Literal,
+        Lambda,
+        LegacyLet,
+        LetIn,
+        List,
+        BinOp,
+        Paren,
+        Root,
+        AttrSet,
+        UnaryOp,
+        Ident,
+        With,
+        HasAttr,
+        CurPos,
+    )]
+    /// An expression. The fundamental nix ast type.
+    enum Expr;
+}
+
+node! {
+    #[from(
+        Ident,
+        Dynamic,
+        Str,
+    )]
+    enum Attr;
+}
+
+node! { #[from(NODE_IDENT)] struct Ident; }
+
+impl Ident {
+    // A NODE_IDENT may either have a nested TOKEN_OR if the identifier was "or" or TOKEN_IDENT for everything else
+    tg! { ident_token, TOKEN_IDENT }
+}
+
+node! { #[from(NODE_APPLY)] struct Apply; }
+
+impl Apply {
+    ng! { lambda, Expr, 0 }
+    ng! { argument, Expr, 1 }
+}
+
+node! { #[from(NODE_ASSERT)] struct Assert; }
+
+impl Assert {
+    tg! { assert_token, assert }
+    ng! { condition, Expr, 0 }
+    ng! { body, Expr, 1 }
+}
+
+node! { #[from(NODE_ATTRPATH)] struct Attrpath; }
+
+impl Attrpath {
+    ng! { attrs, [Attr] }
+}
+
+node! { #[from(NODE_DYNAMIC)] struct Dynamic; }
+
+impl Dynamic {
+    tg! { interpol_start_token, TOKEN_INTERPOL_START }
+    ng! { expr, Expr, 0 }
+    tg! { interpol_end_token, TOKEN_INTERPOL_END }
+}
+
+node! { #[from(NODE_ERROR)] struct Error; }
+
+node! { #[from(NODE_IF_ELSE)] struct IfElse; }
+
+impl IfElse {
+    tg! { if_token, if }
+    ng! { condition, Expr, 0 }
+    tg! { then_token, then }
+    ng! { body, Expr, 1 }
+    tg! { else_token, else }
+    ng! { else_body, Expr, 2 }
+}
+
+node! { #[from(NODE_SELECT)] struct Select; }
+
+impl Select {
+    ng! { expr, Expr, 0 }
+    tg! { dot_token, . }
+    ng! { attrpath, Attrpath, 0 }
+    tg! { or_token, or }
+    ng! { default_expr, Expr, 1 }
+}
+
+node! { #[from(NODE_INHERIT)] struct Inherit; }
+
+impl Inherit {
+    tg! { inherit_token, inherit }
+    ng! { from, InheritFrom, 0 }
+    ng! { attrs, [Attr] }
+}
+
+node! { #[from(NODE_INHERIT_FROM)] struct InheritFrom; }
+
+impl InheritFrom {
+    tg! { l_paren_token, '(' }
+    ng! { expr, Expr, 0 }
+    tg! { r_paren_token, ')' }
+}
+
+node! { #[from(NODE_PATH_ABS)]    struct PathAbs; }
+node! { #[from(NODE_PATH_REL)]    struct PathRel; }
+node! { #[from(NODE_PATH_HOME)]   struct PathHome; }
+node! { #[from(NODE_PATH_SEARCH)] struct PathSearch; }
+
+node! {
+    #[from(
+        PathAbs,
+        PathRel,
+        PathHome,
+        PathSearch,
+    )]
+    enum Path;
+}
+
+node! { #[from(NODE_STRING)] struct Str; }
+
+node! { #[from(NODE_INTERPOL)] struct Interpol; }
+
+impl Interpol {
+    ng! { expr, Expr, 0 }
+}
+
+node! { #[from(NODE_LAMBDA)] struct Lambda; }
+
+impl Lambda {
+    ng! { param, Param, 0 }
+    tg! { token_colon, : }
+    ng! { body, Expr, 0 }
+}
+
+node! { #[from(NODE_LEGACY_LET)] struct LegacyLet; }
+
+impl HasEntry for LegacyLet {}
+
+impl LegacyLet {
+    tg! { let_token, let }
+    tg! { curly_open_token, '{' }
+    tg! { curly_close_token, '}' }
+}
+
+node! { #[from(NODE_LET_IN)] struct LetIn; }
+
+impl HasEntry for LetIn {}
+
+impl LetIn {
+    tg! { let_token, let }
+    tg! { in_token, in }
+    ng! { body, Expr, 0 }
+}
+
+node! { #[from(NODE_LIST)] struct List; }
+
+impl List {
+    tg! { l_brack_token, '[' }
+    ng! { items, [Expr] }
+    tg! { r_brack_token, ']' }
+}
+
+node! { #[from(NODE_BIN_OP)] struct BinOp; }
+
+impl BinOp {
+    ng! { lhs, Expr, 0 }
+
+    pub fn operator(&self) -> Option<BinOpKind> {
+        children_tokens_u(self).find_map(|t| BinOpKind::from_kind(t.kind()))
+    }
+
+    ng! { rhs, Expr, 1 }
+}
+
+node! { #[from(NODE_PAREN)] struct Paren; }
+
+impl Paren {
+    tg! { l_paren_token, '(' }
+    ng! { expr, Expr, 0 }
+    tg! { r_paren_token, ')' }
+}
+
+node! { #[from(NODE_PAT_BIND)] struct PatBind; }
+
+impl PatBind {
+    ng! { ident, Ident, 0 }
+}
+
+node! { #[from(NODE_PAT_ENTRY)] struct PatEntry; }
+
+impl PatEntry {
+    ng! { ident, Ident, 0 }
+    tg! { question_token, ? }
+    ng! { default, Expr, 1 }
+}
+
+node! { #[from(NODE_IDENT_PARAM)] struct IdentParam; }
+
+impl IdentParam {
+    ng! { ident, Ident, 0 }
+}
+
+node! {
+    #[from(
+        Pattern,
+        IdentParam,
+    )]
+    enum Param;
+}
+
+node! { #[from(NODE_PATTERN)] struct Pattern; }
+
+impl Pattern {
+    tg! { at_token, @ }
+    ng! { pat_entries, [PatEntry] }
+    tg! { ellipsis_token, ... }
+    ng! { pat_bind, PatBind, 0 }
+}
+
+node! { #[from(NODE_ROOT)] struct Root; }
+
+impl Root {
+    ng! { expr, Expr, 0 }
+}
+
+node! { #[from(NODE_ATTR_SET)] struct AttrSet; }
+
+impl HasEntry for AttrSet {}
+
+impl AttrSet {
+    tg! { rec_token, rec }
+    tg! { l_curly_token, '{' }
+    tg! { r_curly_token, '}' }
+}
+
+node! {
+    #[from(
+        Inherit,
+        AttrpathValue,
+    )]
+    enum Entry;
+}
+
+node! { #[from(NODE_ATTRPATH_VALUE)] struct AttrpathValue; }
+
+impl AttrpathValue {
+    ng! { attrpath, Attrpath, 0 }
+    tg! { assign_token, = }
+    ng! { value, Expr, 0 }
+}
+
+node! { #[from(NODE_UNARY_OP)] struct UnaryOp; }
+
+impl UnaryOp {
+    pub fn operator(&self) -> Option<UnaryOpKind> {
+        children_tokens_u(self).find_map(|t| UnaryOpKind::from_kind(t.kind()))
+    }
+    ng! { expr, Expr, 0 }
+}
+
+node! { #[from(NODE_WITH)] struct With; }
+
+impl With {
+    tg! { with_token, with }
+    ng! { namespace, Expr, 0 }
+    tg! { semicolon_token, ; }
+    ng! { body, Expr, 1 }
+}
+
+node! { #[from(NODE_HAS_ATTR)] struct HasAttr; }
+
+impl HasAttr {
+    ng! { expr, Expr, 0 }
+    tg! { question_token, ? }
+    ng! { attrpath, Attrpath, 0 }
+}
+
+node! { #[from(NODE_CUR_POS)] struct CurPos; }
+
+impl CurPos {
+    tg! { cur_pos_token, __curPos }
+}
@@ -0,0 +1,71 @@
+use crate::SyntaxKind::{self, *};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum BinOpKind {
+    Concat,
+    Update,
+
+    Add,
+    Sub,
+    Mul,
+    Div,
+
+    And,
+    Equal,
+    Implication,
+    Less,
+    LessOrEq,
+    More,
+    MoreOrEq,
+    NotEqual,
+    Or,
+    PipeRight,
+    PipeLeft,
+}
+
+impl BinOpKind {
+    /// Get the operation kind from a SyntaxKind in the AST
+    pub fn from_kind(token: SyntaxKind) -> Option<Self> {
+        match token {
+            TOKEN_CONCAT => Some(BinOpKind::Concat),
+            TOKEN_UPDATE => Some(BinOpKind::Update),
+
+            TOKEN_ADD => Some(BinOpKind::Add),
+            TOKEN_SUB => Some(BinOpKind::Sub),
+            TOKEN_MUL => Some(BinOpKind::Mul),
+            TOKEN_DIV => Some(BinOpKind::Div),
+
+            TOKEN_AND_AND => Some(BinOpKind::And),
+            TOKEN_EQUAL => Some(BinOpKind::Equal),
+            TOKEN_IMPLICATION => Some(BinOpKind::Implication),
+            TOKEN_LESS => Some(BinOpKind::Less),
+            TOKEN_LESS_OR_EQ => Some(BinOpKind::LessOrEq),
+            TOKEN_MORE => Some(BinOpKind::More),
+            TOKEN_MORE_OR_EQ => Some(BinOpKind::MoreOrEq),
+            TOKEN_NOT_EQUAL => Some(BinOpKind::NotEqual),
+            TOKEN_OR_OR => Some(BinOpKind::Or),
+
+            TOKEN_PIPE_RIGHT => Some(BinOpKind::PipeRight),
+            TOKEN_PIPE_LEFT => Some(BinOpKind::PipeLeft),
+
+            _ => None,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum UnaryOpKind {
+    Invert,
+    Negate,
+}
+
+impl UnaryOpKind {
+    /// Get the operation kind from a token in the AST
+    pub fn from_kind(kind: SyntaxKind) -> Option<Self> {
+        match kind {
+            TOKEN_INVERT => Some(UnaryOpKind::Invert),
+            TOKEN_SUB => Some(UnaryOpKind::Negate),
+            _ => None,
+        }
+    }
+}
@@ -0,0 +1,204 @@
+use crate::kinds::SyntaxKind::{TOKEN_PATH_ABS, TOKEN_PATH_HOME, TOKEN_PATH_REL};
+
+use rowan::{ast::AstNode as OtherAstNode, NodeOrToken};
+
+pub use super::nodes::Path;
+use super::{
+    nodes::{PathAbs, PathHome, PathRel, PathSearch},
+    AstToken, InterpolPart, PathContent,
+};
+use crate::ast;
+
+fn extract_path_parts<T: ast::AstNode>(node: &T) -> Vec<InterpolPart<PathContent>> {
+    node.syntax()
+        .children_with_tokens()
+        .map(|child| match child {
+            NodeOrToken::Token(token) => {
+                debug_assert!(matches!(
+                    token.kind(),
+                    TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME
+                ));
+                InterpolPart::Literal(PathContent::cast(token).unwrap())
+            }
+            NodeOrToken::Node(node) => {
+                InterpolPart::Interpolation(ast::Interpol::cast(node).unwrap())
+            }
+        })
+        .collect()
+}
+
+// Direct methods for interpolatable path types
+impl PathAbs {
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        extract_path_parts(self)
+    }
+}
+
+impl PathRel {
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        extract_path_parts(self)
+    }
+}
+
+impl PathHome {
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        extract_path_parts(self)
+    }
+}
+
+// Direct methods for search path
+impl PathSearch {
+    /// Get the content of a search path
+    pub fn content(&self) -> Option<PathContent> {
+        self.syntax()
+            .children_with_tokens()
+            .filter_map(|child| child.into_token().and_then(PathContent::cast))
+            .next()
+    }
+}
+
+/// Extension methods for the Path enum
+impl Path {
+    /// Get parts from any path type in a unified way
+    pub fn parts(&self) -> Vec<InterpolPart<PathContent>> {
+        match self {
+            // For interpolatable paths, get their parts
+            Path::PathAbs(p) => p.parts(),
+            Path::PathRel(p) => p.parts(),
+            Path::PathHome(p) => p.parts(),
+            // For search paths, return a single literal component if content exists
+            Path::PathSearch(p) => {
+                if let Some(content) = p.content() {
+                    vec![InterpolPart::Literal(content)]
+                } else {
+                    vec![]
+                }
+            }
+        }
+    }
+
+    pub fn is_search(&self) -> bool {
+        matches!(self, Path::PathSearch(_))
+    }
+
+    pub fn is_interpolatable(&self) -> bool {
+        !self.is_search()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rowan::ast::AstNode;
+
+    use super::InterpolPart;
+    use crate::{
+        ast::{self, Path},
+        Root,
+    };
+
+    #[test]
+    fn test_path_types() {
+        // Absolute path
+        let inp = "/foo/bar";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathAbs(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+            assert!(path.is_interpolatable());
+            assert!(!path.is_search());
+        }
+
+        // Search path
+        let inp = "<nixpkgs>";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathSearch(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+            assert!(!path.is_interpolatable());
+            assert!(path.is_search());
+        }
+    }
+
+    #[test]
+    fn test_parts() {
+        // Test parts with absolute path
+        let inp = "/foo/bar";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathAbs(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+
+            let parts = path.parts();
+            assert_eq!(parts.len(), 1);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "/foo/bar");
+                }
+                _ => panic!("Expected literal part"),
+            }
+        }
+
+        // Test parts with interpolated path
+        let inp = r#"./a/${"hello"}"#;
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathRel(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+
+            let parts = path.parts();
+            assert_eq!(parts.len(), 2);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "./a/");
+                }
+                _ => panic!("Expected literal part"),
+            }
+
+            match &parts[1] {
+                InterpolPart::Interpolation(_) => {} // Success
+                _ => panic!("Expected interpolation part"),
+            }
+        }
+
+        // Test parts with search path
+        let inp = "<nixpkgs>";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathSearch(p) = expr {
+            let path = Path::cast(p.syntax().clone()).unwrap();
+
+            let parts = path.parts();
+            assert_eq!(parts.len(), 1);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "<nixpkgs>");
+                }
+                _ => panic!("Expected literal part"),
+            }
+        }
+    }
+
+    #[test]
+    fn direct_method_usage() {
+        // Test direct parts() method on PathAbs
+        let inp = "/foo/bar";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathAbs(p) = expr {
+            let parts = p.parts();
+            assert_eq!(parts.len(), 1);
+
+            match &parts[0] {
+                InterpolPart::Literal(content) => {
+                    assert_eq!(content.text(), "/foo/bar");
+                }
+                _ => panic!("Expected literal part"),
+            }
+        }
+
+        // Test direct content() method on PathSearch
+        let inp = "<nixpkgs>";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        if let ast::Expr::PathSearch(p) = expr {
+            let content = p.content().expect("Expected content");
+            assert_eq!(content.text(), "<nixpkgs>");
+        }
+    }
+}
@@ -0,0 +1,363 @@
+use crate::kinds::SyntaxKind::*;
+use rowan::{ast::AstNode as OtherAstNode, NodeOrToken};
+
+use crate::ast;
+
+use super::{support::children_tokens_u, AstToken, InterpolPart, StrContent};
+
+impl ast::Str {
+    pub fn parts(&self) -> impl Iterator<Item = InterpolPart<StrContent>> {
+        self.syntax().children_with_tokens().filter_map(|child| match child {
+            NodeOrToken::Token(token) if token.kind() == TOKEN_STRING_CONTENT => {
+                Some(InterpolPart::Literal(StrContent::cast(token).unwrap()))
+            }
+            NodeOrToken::Token(token) => {
+                assert!(token.kind() == TOKEN_STRING_START || token.kind() == TOKEN_STRING_END);
+                None
+            }
+            NodeOrToken::Node(node) => {
+                assert_eq!(node.kind(), NODE_INTERPOL);
+                Some(InterpolPart::Interpolation(ast::Interpol::cast(node).unwrap()))
+            }
+        })
+    }
+
+    pub fn normalized_parts(&self) -> Vec<InterpolPart<String>> {
+        let multiline = children_tokens_u(self).next().is_some_and(|t| t.text() == "''");
+        let mut is_first_literal = true;
+        let mut at_start_of_line = true;
+        let mut min_indent = 1000000;
+        let mut cur_indent = 0;
+        let mut n = 0;
+        let mut first_is_literal = false;
+
+        let parts: Vec<InterpolPart<StrContent>> = self.parts().collect();
+
+        if multiline {
+            for part in &parts {
+                match part {
+                    InterpolPart::Interpolation(_) => {
+                        if at_start_of_line {
+                            at_start_of_line = false;
+                            min_indent = min_indent.min(cur_indent);
+                        }
+                        n += 1;
+                    }
+                    InterpolPart::Literal(literal) => {
+                        let mut token_text = literal.syntax().text();
+
+                        if n == 0 {
+                            first_is_literal = true;
+                        }
+
+                        if is_first_literal && first_is_literal {
+                            is_first_literal = false;
+                            if let Some(p) = token_text.find('\n') {
+                                if token_text[0..p].chars().all(|c| c == ' ') {
+                                    token_text = &token_text[p + 1..]
+                                }
+                            }
+                        }
+
+                        for c in token_text.chars() {
+                            if at_start_of_line {
+                                if c == ' ' {
+                                    cur_indent += 1;
+                                } else if c == '\n' {
+                                    cur_indent = 0;
+                                } else {
+                                    at_start_of_line = false;
+                                    min_indent = min_indent.min(cur_indent);
+                                }
+                            } else if c == '\n' {
+                                at_start_of_line = true;
+                                cur_indent = 0;
+                            }
+                        }
+
+                        n += 1;
+                    }
+                }
+            }
+        }
+
+        let mut normalized_parts = Vec::new();
+        let mut cur_dropped = 0;
+        let mut i = 0;
+        is_first_literal = true;
+        at_start_of_line = true;
+
+        for part in parts {
+            match part {
+                InterpolPart::Interpolation(interpol) => {
+                    at_start_of_line = false;
+                    cur_dropped = 0;
+                    normalized_parts.push(InterpolPart::Interpolation(interpol));
+                    i += 1;
+                }
+                InterpolPart::Literal(literal) => {
+                    let mut token_text = literal.syntax().text();
+
+                    if multiline {
+                        if is_first_literal && first_is_literal {
+                            is_first_literal = false;
+                            if let Some(p) = token_text.find('\n') {
+                                if token_text[0..p].chars().all(|c| c == ' ') {
+                                    token_text = &token_text[p + 1..];
+                                    if token_text.is_empty() {
+                                        i += 1;
+                                        continue;
+                                    }
+                                }
+                            }
+                        }
+
+                        let mut str = String::new();
+                        for c in token_text.chars() {
+                            if at_start_of_line {
+                                if c == ' ' {
+                                    if cur_dropped >= min_indent {
+                                        str.push(c);
+                                    }
+                                    cur_dropped += 1;
+                                } else if c == '\n' {
+                                    cur_dropped = 0;
+                                    str.push(c);
+                                } else {
+                                    at_start_of_line = false;
+                                    cur_dropped = 0;
+                                    str.push(c);
+                                }
+                            } else {
+                                str.push(c);
+                                if c == '\n' {
+                                    at_start_of_line = true;
+                                }
+                            }
+                        }
+
+                        if i == n - 1 {
+                            if let Some(p) = str.rfind('\n') {
+                                if str[p + 1..].chars().all(|c| c == ' ') {
+                                    str.truncate(p + 1);
+                                }
+                            }
+                        }
+
+                        normalized_parts.push(InterpolPart::Literal(unescape(&str, multiline)));
+                        i += 1;
+                    } else {
+                        normalized_parts
+                            .push(InterpolPart::Literal(unescape(token_text, multiline)));
+                    }
+                }
+            }
+        }
+
+        normalized_parts
+    }
+}
+
+/// Interpret escape sequences in the nix string and return the converted value
+pub fn unescape(input: &str, multiline: bool) -> String {
+    let mut output = String::new();
+    let mut input = input.chars().peekable();
+    loop {
+        match input.next() {
+            None => break,
+            Some('"') if !multiline => break,
+            Some('\\') if !multiline => match input.next() {
+                None => break,
+                Some('n') => output.push('\n'),
+                Some('r') => output.push('\r'),
+                Some('t') => output.push('\t'),
+                Some(c) => output.push(c),
+            },
+            Some('\'') if multiline => match input.next() {
+                None => {
+                    output.push('\'');
+                }
+                Some('\'') => match input.peek() {
+                    Some('\'') => {
+                        input.next().unwrap();
+                        output.push_str("''");
+                    }
+                    Some('$') => {
+                        input.next().unwrap();
+                        output.push('$');
+                    }
+                    Some('\\') => {
+                        input.next().unwrap();
+                        match input.next() {
+                            None => break,
+                            Some('n') => output.push('\n'),
+                            Some('r') => output.push('\r'),
+                            Some('t') => output.push('\t'),
+                            Some(c) => output.push(c),
+                        }
+                    }
+                    _ => break,
+                },
+                Some(c) => {
+                    output.push('\'');
+                    output.push(c);
+                }
+            },
+            Some(c) => output.push(c),
+        }
+    }
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::Root;
+
+    use super::*;
+
+    #[test]
+    fn string_unescapes() {
+        assert_eq!(unescape(r#"Hello\n\"World\" :D"#, false), "Hello\n\"World\" :D");
+        assert_eq!(unescape(r#"\"Hello\""#, false), "\"Hello\"");
+
+        assert_eq!(unescape(r#"Hello''\n'''World''' :D"#, true), "Hello\n''World'' :D");
+        assert_eq!(unescape(r#""Hello""#, true), "\"Hello\"");
+    }
+    #[test]
+    fn parts_leading_ws() {
+        let inp = "''\n  hello\n  world''";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(str) => {
+                assert_eq!(
+                    str.normalized_parts(),
+                    vec![InterpolPart::Literal("hello\nworld".to_string())]
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+    #[test]
+    fn parts_trailing_ws_single_line() {
+        let inp = "''hello ''";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(str) => {
+                assert_eq!(
+                    str.normalized_parts(),
+                    vec![InterpolPart::Literal("hello ".to_string())]
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+    #[test]
+    fn parts_trailing_ws_multiline() {
+        let inp = "''hello\n ''";
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(str) => {
+                assert_eq!(
+                    str.normalized_parts(),
+                    vec![InterpolPart::Literal("hello\n".to_string())]
+                )
+            }
+            _ => unreachable!(),
+        }
+    }
+    #[test]
+    fn parts() {
+        use crate::{NixLanguage, SyntaxNode};
+        use rowan::{GreenNodeBuilder, Language};
+
+        fn string_node(content: &str) -> ast::Str {
+            let mut builder = GreenNodeBuilder::new();
+            builder.start_node(NixLanguage::kind_to_raw(NODE_STRING));
+            builder.token(NixLanguage::kind_to_raw(TOKEN_STRING_START), "''");
+            builder.token(NixLanguage::kind_to_raw(TOKEN_STRING_CONTENT), content);
+            builder.token(NixLanguage::kind_to_raw(TOKEN_STRING_END), "''");
+            builder.finish_node();
+
+            ast::Str::cast(SyntaxNode::new_root(builder.finish())).unwrap()
+        }
+
+        let txtin = r#"
+                        |trailing-whitespace
+                              |trailing-whitespace
+                    This is a multiline string :D
+                      indented by two
+                    \'\'\'\'\
+                    ''${ interpolation was escaped }
+                    two single quotes: '''
+                    three single quotes: ''''
+                "#
+        .replace("|trailing-whitespace", "");
+
+        if let [InterpolPart::Literal(lit)] =
+            &ast::Str::normalized_parts(&string_node(txtin.as_str()))[..]
+        {
+            assert_eq!(lit,
+                // Get the below with nix repl
+                "    \n          \nThis is a multiline string :D\n  indented by two\n\\'\\'\\'\\'\\\n${ interpolation was escaped }\ntwo single quotes: ''\nthree single quotes: '''\n"
+            );
+        } else {
+            unreachable!();
+        }
+    }
+
+    #[test]
+    fn parts_ast() {
+        fn assert_eq_ast_ctn(it: &mut dyn Iterator<Item = InterpolPart<String>>, x: &str) {
+            let tmp = it.next().expect("unexpected EOF");
+            if let InterpolPart::Interpolation(astn) = tmp {
+                assert_eq!(astn.expr().unwrap().syntax().to_string(), x);
+            } else {
+                unreachable!("unexpected literal {:?}", tmp);
+            }
+        }
+
+        let inp = r#"''
+
+        This version of Nixpkgs requires Nix >= ${requiredVersion}, please upgrade:
+
+        - If you are running NixOS, `nixos-rebuild' can be used to upgrade your system.
+
+        - Alternatively, with Nix > 2.0 `nix upgrade-nix' can be used to imperatively
+          upgrade Nix. You may use `nix-env --version' to check which version you have.
+
+        - If you installed Nix using the install script (https://nixos.org/nix/install),
+          it is safe to upgrade by running it again:
+
+              curl -L https://nixos.org/nix/install | sh
+
+        For more information, please see the NixOS release notes at
+        https://nixos.org/nixos/manual or locally at
+        ${toString ./nixos/doc/manual/release-notes}.
+
+        If you need further help, see https://nixos.org/nixos/support.html
+      ''"#;
+        let expr = Root::parse(inp).ok().unwrap().expr().unwrap();
+        match expr {
+            ast::Expr::Str(s) => {
+                let mut it = s.normalized_parts().into_iter();
+                assert_eq!(
+                    it.next().unwrap(),
+                    InterpolPart::Literal("\nThis version of Nixpkgs requires Nix >= ".to_string())
+                );
+                assert_eq_ast_ctn(&mut it, "requiredVersion");
+                assert_eq!(it.next().unwrap(), InterpolPart::Literal(
+                        ", please upgrade:\n\n- If you are running NixOS, `nixos-rebuild' can be used to upgrade your system.\n\n- Alternatively, with Nix > 2.0 `nix upgrade-nix' can be used to imperatively\n  upgrade Nix. You may use `nix-env --version' to check which version you have.\n\n- If you installed Nix using the install script (https://nixos.org/nix/install),\n  it is safe to upgrade by running it again:\n\n      curl -L https://nixos.org/nix/install | sh\n\nFor more information, please see the NixOS release notes at\nhttps://nixos.org/nixos/manual or locally at\n".to_string()
+                    ));
+                assert_eq_ast_ctn(&mut it, "toString ./nixos/doc/manual/release-notes");
+                assert_eq!(
+                    it.next().unwrap(),
+                    InterpolPart::Literal(
+                        ".\n\nIf you need further help, see https://nixos.org/nixos/support.html\n"
+                            .to_string()
+                    )
+                );
+            }
+            _ => unreachable!(),
+        }
+    }
+}
@@ -0,0 +1,174 @@
+use core::num;
+
+use crate::{
+    ast::AstToken,
+    SyntaxKind::{self, *},
+    SyntaxToken,
+};
+
+macro_rules! token {
+    (
+        #[from($kind:ident)]
+        $(#[$meta:meta])*
+        struct $name:ident;
+    ) => {
+        #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+        $(#[$meta])*
+        pub struct $name(pub(super) SyntaxToken);
+
+        impl std::fmt::Display for $name {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                std::fmt::Display::fmt(self.syntax(), f)
+            }
+        }
+
+        impl AstToken for $name {
+            fn can_cast(kind: SyntaxKind) -> bool {
+                $kind == kind
+            }
+
+            fn cast(from: SyntaxToken) -> Option<Self> {
+                if from.kind() == $kind {
+                    Some(Self(from))
+                } else {
+                    None
+                }
+            }
+
+            fn syntax(&self) -> &SyntaxToken {
+                &self.0
+            }
+        }
+    };
+}
+
+token! { #[from(TOKEN_WHITESPACE)] struct Whitespace; }
+
+token! { #[from(TOKEN_COMMENT)] struct Comment; }
+
+impl Comment {
+    pub fn text(&self) -> &str {
+        let text = self.syntax().text();
+        // Handle both "#..." and "/*...*/" comments.
+        match text.strip_prefix('#') {
+            Some(s) => s,
+            None => text.strip_prefix(r#"/*"#).unwrap().strip_suffix(r#"*/"#).unwrap(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::ast;
+    use crate::match_ast;
+    use crate::Root;
+
+    use super::*;
+    use rowan::ast::AstNode;
+
+    #[test]
+    fn comment() {
+        let s = "# comment bruh
+/* this is a multiline comment wow
+asdfasdf
+asdfasdf */
+1 + 1
+/* last one */
+";
+        let comments: Vec<String> = Root::parse(s)
+            .ok()
+            .unwrap()
+            .syntax()
+            .children_with_tokens()
+            .filter_map(|e| match e {
+                rowan::NodeOrToken::Token(token) => match_ast! { match token {
+                    ast::Comment(it) => Some(it.text().to_string()),
+                    _ => None,
+                }},
+                rowan::NodeOrToken::Node(_) => None,
+            })
+            .collect();
+        let expected = vec![
+            " comment bruh",
+            " this is a multiline comment wow\nasdfasdf\nasdfasdf ",
+            " last one ",
+        ];
+
+        assert_eq!(comments, expected);
+    }
+}
+
+token! { #[from(TOKEN_FLOAT)] struct Float; }
+
+impl Float {
+    pub fn value(&self) -> Result<f64, num::ParseFloatError> {
+        self.syntax().text().parse()
+    }
+}
+
+token! { #[from(TOKEN_INTEGER)] struct Integer; }
+
+impl Integer {
+    pub fn value(&self) -> Result<i64, num::ParseIntError> {
+        self.syntax().text().parse()
+    }
+}
+
+// A literal part of a path (absolute / relative / home / search) without
+// distinguishing which concrete flavour it was taken from.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PathContent(pub(super) SyntaxToken);
+
+impl PathContent {
+    /// Returns the path as a string
+    pub fn text(&self) -> &str {
+        self.syntax().text()
+    }
+
+    /// Returns the kind of path token
+    pub fn path_kind(&self) -> SyntaxKind {
+        self.syntax().kind()
+    }
+
+    /// Returns true if this is an absolute path
+    pub fn is_absolute(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_ABS
+    }
+
+    /// Returns true if this is a relative path
+    pub fn is_relative(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_REL
+    }
+
+    /// Returns true if this is a home-relative path
+    pub fn is_home(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_HOME
+    }
+
+    /// Returns true if this is a search path
+    pub fn is_search(&self) -> bool {
+        self.path_kind() == TOKEN_PATH_SEARCH
+    }
+}
+
+impl AstToken for PathContent {
+    fn can_cast(kind: SyntaxKind) -> bool {
+        matches!(kind, TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME | TOKEN_PATH_SEARCH)
+    }
+
+    fn cast(from: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(from.kind()) {
+            Some(Self(from))
+        } else {
+            None
+        }
+    }
+
+    fn syntax(&self) -> &SyntaxToken {
+        &self.0
+    }
+}
+
+token! { #[from(TOKEN_STRING_CONTENT)] struct StrContent; }
+
+token! { #[from(TOKEN_URI)] struct Uri; }
@@ -0,0 +1,285 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[repr(u16)]
+#[allow(non_camel_case_types)]
+#[non_exhaustive]
+pub enum SyntaxKind {
+    TOKEN_ADD,
+    TOKEN_AND_AND,
+    TOKEN_ASSIGN,
+    TOKEN_ASSERT,
+    TOKEN_AT,
+    TOKEN_COLON,
+    TOKEN_COMMA,
+    TOKEN_COMMENT,
+    TOKEN_CONCAT,
+    TOKEN_CUR_POS,
+    TOKEN_DIV,
+    TOKEN_DOT,
+    TOKEN_ELLIPSIS,
+    TOKEN_ELSE,
+    TOKEN_EQUAL,
+    TOKEN_ERROR,
+    TOKEN_FLOAT,
+    TOKEN_IDENT,
+    TOKEN_IF,
+    TOKEN_IMPLICATION,
+    TOKEN_IN,
+    TOKEN_INHERIT,
+    TOKEN_INTEGER,
+    TOKEN_INTERPOL_END,
+    TOKEN_INTERPOL_START,
+    TOKEN_INVERT,
+    TOKEN_L_BRACE,
+    TOKEN_L_BRACK,
+    TOKEN_L_PAREN,
+    TOKEN_LESS,
+    TOKEN_LESS_OR_EQ,
+    TOKEN_LET,
+    TOKEN_MORE,
+    TOKEN_MORE_OR_EQ,
+    TOKEN_MUL,
+    TOKEN_NOT_EQUAL,
+    TOKEN_OR,
+    TOKEN_OR_OR,
+    TOKEN_PATH_ABS,
+    TOKEN_PATH_HOME,
+    TOKEN_PATH_REL,
+    TOKEN_PATH_SEARCH,
+    TOKEN_PIPE_LEFT,
+    TOKEN_PIPE_RIGHT,
+    TOKEN_QUESTION,
+    TOKEN_R_BRACE,
+    TOKEN_R_BRACK,
+    TOKEN_R_PAREN,
+    TOKEN_REC,
+    TOKEN_SEMICOLON,
+    TOKEN_STRING_CONTENT,
+    TOKEN_STRING_END,
+    TOKEN_STRING_START,
+    TOKEN_SUB,
+    TOKEN_THEN,
+    TOKEN_UPDATE,
+    TOKEN_URI,
+    TOKEN_WHITESPACE,
+    TOKEN_WITH,
+
+    NODE_APPLY,
+    NODE_ASSERT,
+    NODE_ATTR_SET,
+    NODE_ATTRPATH,
+    NODE_ATTRPATH_VALUE,
+    NODE_BIN_OP,
+    NODE_CUR_POS,
+    NODE_DYNAMIC,
+    NODE_ERROR,
+    // Attrpath existence check: foo ? bar.${baz}."bux"
+    NODE_HAS_ATTR,
+    NODE_IDENT,
+    NODE_IDENT_PARAM,
+    NODE_IF_ELSE,
+    NODE_INHERIT,
+    NODE_INHERIT_FROM,
+    NODE_INTERPOL,
+    NODE_LAMBDA,
+    // An old let { x = 92; body = x; } syntax
+    NODE_LEGACY_LET,
+    NODE_LET_IN,
+    NODE_LIST,
+    NODE_LITERAL,
+    NODE_PAREN,
+    NODE_PAT_BIND,
+    NODE_PAT_ENTRY,
+    NODE_PATTERN,
+    NODE_PATH_ABS,
+    NODE_PATH_HOME,
+    NODE_PATH_REL,
+    NODE_PATH_SEARCH,
+    NODE_ROOT,
+    NODE_SELECT,
+    NODE_STRING,
+    NODE_UNARY_OP,
+    NODE_WITH,
+}
+use SyntaxKind::*;
+
+/// Maximum valid discriminant value for SyntaxKind
+pub(crate) const SYNTAX_KIND_MAX: u16 = NODE_WITH as u16;
+
+/// Error returned when parsing a string into a `SyntaxKind` fails.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ParseSyntaxKindError;
+
+impl std::fmt::Display for ParseSyntaxKindError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "invalid SyntaxKind")
+    }
+}
+
+impl std::error::Error for ParseSyntaxKindError {}
+
+impl std::str::FromStr for SyntaxKind {
+    type Err = ParseSyntaxKindError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "TOKEN_ADD" => Ok(TOKEN_ADD),
+            "TOKEN_AND_AND" => Ok(TOKEN_AND_AND),
+            "TOKEN_ASSIGN" => Ok(TOKEN_ASSIGN),
+            "TOKEN_ASSERT" => Ok(TOKEN_ASSERT),
+            "TOKEN_AT" => Ok(TOKEN_AT),
+            "TOKEN_COLON" => Ok(TOKEN_COLON),
+            "TOKEN_COMMA" => Ok(TOKEN_COMMA),
+            "TOKEN_COMMENT" => Ok(TOKEN_COMMENT),
+            "TOKEN_CONCAT" => Ok(TOKEN_CONCAT),
+            "TOKEN_CUR_POS" => Ok(TOKEN_CUR_POS),
+            "TOKEN_DIV" => Ok(TOKEN_DIV),
+            "TOKEN_DOT" => Ok(TOKEN_DOT),
+            "TOKEN_ELLIPSIS" => Ok(TOKEN_ELLIPSIS),
+            "TOKEN_ELSE" => Ok(TOKEN_ELSE),
+            "TOKEN_EQUAL" => Ok(TOKEN_EQUAL),
+            "TOKEN_ERROR" => Ok(TOKEN_ERROR),
+            "TOKEN_FLOAT" => Ok(TOKEN_FLOAT),
+            "TOKEN_IDENT" => Ok(TOKEN_IDENT),
+            "TOKEN_IF" => Ok(TOKEN_IF),
+            "TOKEN_IMPLICATION" => Ok(TOKEN_IMPLICATION),
+            "TOKEN_IN" => Ok(TOKEN_IN),
+            "TOKEN_INHERIT" => Ok(TOKEN_INHERIT),
+            "TOKEN_INTEGER" => Ok(TOKEN_INTEGER),
+            "TOKEN_INTERPOL_END" => Ok(TOKEN_INTERPOL_END),
+            "TOKEN_INTERPOL_START" => Ok(TOKEN_INTERPOL_START),
+            "TOKEN_INVERT" => Ok(TOKEN_INVERT),
+            "TOKEN_L_BRACE" => Ok(TOKEN_L_BRACE),
+            "TOKEN_L_BRACK" => Ok(TOKEN_L_BRACK),
+            "TOKEN_L_PAREN" => Ok(TOKEN_L_PAREN),
+            "TOKEN_LESS" => Ok(TOKEN_LESS),
+            "TOKEN_LESS_OR_EQ" => Ok(TOKEN_LESS_OR_EQ),
+            "TOKEN_LET" => Ok(TOKEN_LET),
+            "TOKEN_MORE" => Ok(TOKEN_MORE),
+            "TOKEN_MORE_OR_EQ" => Ok(TOKEN_MORE_OR_EQ),
+            "TOKEN_MUL" => Ok(TOKEN_MUL),
+            "TOKEN_NOT_EQUAL" => Ok(TOKEN_NOT_EQUAL),
+            "TOKEN_OR" => Ok(TOKEN_OR),
+            "TOKEN_OR_OR" => Ok(TOKEN_OR_OR),
+            "TOKEN_PATH_ABS" => Ok(TOKEN_PATH_ABS),
+            "TOKEN_PATH_HOME" => Ok(TOKEN_PATH_HOME),
+            "TOKEN_PATH_REL" => Ok(TOKEN_PATH_REL),
+            "TOKEN_PATH_SEARCH" => Ok(TOKEN_PATH_SEARCH),
+            "TOKEN_PIPE_LEFT" => Ok(TOKEN_PIPE_LEFT),
+            "TOKEN_PIPE_RIGHT" => Ok(TOKEN_PIPE_RIGHT),
+            "TOKEN_QUESTION" => Ok(TOKEN_QUESTION),
+            "TOKEN_R_BRACE" => Ok(TOKEN_R_BRACE),
+            "TOKEN_R_BRACK" => Ok(TOKEN_R_BRACK),
+            "TOKEN_R_PAREN" => Ok(TOKEN_R_PAREN),
+            "TOKEN_REC" => Ok(TOKEN_REC),
+            "TOKEN_SEMICOLON" => Ok(TOKEN_SEMICOLON),
+            "TOKEN_STRING_CONTENT" => Ok(TOKEN_STRING_CONTENT),
+            "TOKEN_STRING_END" => Ok(TOKEN_STRING_END),
+            "TOKEN_STRING_START" => Ok(TOKEN_STRING_START),
+            "TOKEN_SUB" => Ok(TOKEN_SUB),
+            "TOKEN_THEN" => Ok(TOKEN_THEN),
+            "TOKEN_UPDATE" => Ok(TOKEN_UPDATE),
+            "TOKEN_URI" => Ok(TOKEN_URI),
+            "TOKEN_WHITESPACE" => Ok(TOKEN_WHITESPACE),
+            "TOKEN_WITH" => Ok(TOKEN_WITH),
+
+            "NODE_APPLY" => Ok(NODE_APPLY),
+            "NODE_ASSERT" => Ok(NODE_ASSERT),
+            "NODE_ATTR_SET" => Ok(NODE_ATTR_SET),
+            "NODE_ATTRPATH" => Ok(NODE_ATTRPATH),
+            "NODE_ATTRPATH_VALUE" => Ok(NODE_ATTRPATH_VALUE),
+            "NODE_BIN_OP" => Ok(NODE_BIN_OP),
+            "NODE_CUR_POS" => Ok(NODE_CUR_POS),
+            "NODE_DYNAMIC" => Ok(NODE_DYNAMIC),
+            "NODE_ERROR" => Ok(NODE_ERROR),
+            "NODE_HAS_ATTR" => Ok(NODE_HAS_ATTR),
+            "NODE_IDENT" => Ok(NODE_IDENT),
+            "NODE_IDENT_PARAM" => Ok(NODE_IDENT_PARAM),
+            "NODE_IF_ELSE" => Ok(NODE_IF_ELSE),
+            "NODE_INHERIT" => Ok(NODE_INHERIT),
+            "NODE_INHERIT_FROM" => Ok(NODE_INHERIT_FROM),
+            "NODE_INTERPOL" => Ok(NODE_INTERPOL),
+            "NODE_LAMBDA" => Ok(NODE_LAMBDA),
+            "NODE_LEGACY_LET" => Ok(NODE_LEGACY_LET),
+            "NODE_LET_IN" => Ok(NODE_LET_IN),
+            "NODE_LIST" => Ok(NODE_LIST),
+            "NODE_LITERAL" => Ok(NODE_LITERAL),
+            "NODE_PAREN" => Ok(NODE_PAREN),
+            "NODE_PAT_BIND" => Ok(NODE_PAT_BIND),
+            "NODE_PAT_ENTRY" => Ok(NODE_PAT_ENTRY),
+            "NODE_PATTERN" => Ok(NODE_PATTERN),
+            "NODE_PATH_ABS" => Ok(NODE_PATH_ABS),
+            "NODE_PATH_HOME" => Ok(NODE_PATH_HOME),
+            "NODE_PATH_REL" => Ok(NODE_PATH_REL),
+            "NODE_PATH_SEARCH" => Ok(NODE_PATH_SEARCH),
+            "NODE_ROOT" => Ok(NODE_ROOT),
+            "NODE_SELECT" => Ok(NODE_SELECT),
+            "NODE_STRING" => Ok(NODE_STRING),
+            "NODE_UNARY_OP" => Ok(NODE_UNARY_OP),
+            "NODE_WITH" => Ok(NODE_WITH),
+
+            _ => Err(ParseSyntaxKindError),
+        }
+    }
+}
+
+impl SyntaxKind {
+    /// Returns true if this token is a literal, such as an integer or a string
+    pub fn is_literal(self) -> bool {
+        matches!(
+            self,
+            TOKEN_FLOAT
+                | TOKEN_INTEGER
+                | TOKEN_PATH_ABS
+                | TOKEN_PATH_REL
+                | TOKEN_PATH_HOME
+                | TOKEN_PATH_SEARCH
+                | TOKEN_URI
+        )
+    }
+
+    /// Returns true if this token should be used as a function argument.
+    /// ```ignore
+    /// Example:
+    /// add 1 2 + 3
+    /// ^   ^ ^ ^
+    /// |   | | +- false
+    /// |   | +- true
+    /// |   +- true
+    /// +- true
+    /// ```
+    pub fn is_fn_arg(self) -> bool {
+        match self {
+            // NB: the tokenizer always emits TOKEN_IDENT for __curPos;
+            // TOKEN_CUR_POS is included for completeness but cannot
+            // currently be produced by the tokenizer.
+            TOKEN_REC | TOKEN_L_BRACE | TOKEN_L_BRACK | TOKEN_L_PAREN | TOKEN_STRING_START
+            | TOKEN_IDENT | TOKEN_CUR_POS => true,
+            _ => self.is_literal(),
+        }
+    }
+    /// Returns true if this token is a comment, whitespace, or similar, and
+    /// should be skipped over by the parser.
+    pub fn is_trivia(self) -> bool {
+        matches!(self, TOKEN_COMMENT | TOKEN_WHITESPACE)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::str::FromStr;
+
+    #[test]
+    fn from_str_valid() {
+        assert_eq!(SyntaxKind::from_str("TOKEN_COMMENT").unwrap(), TOKEN_COMMENT);
+        assert_eq!(SyntaxKind::from_str("NODE_ROOT").unwrap(), NODE_ROOT);
+        assert_eq!(SyntaxKind::from_str("NODE_HAS_ATTR").unwrap(), NODE_HAS_ATTR);
+    }
+
+    #[test]
+    fn from_str_invalid() {
+        assert!(SyntaxKind::from_str("INVALID").is_err());
+        assert!(SyntaxKind::from_str("").is_err());
+        assert!(SyntaxKind::from_str("token_comment").is_err()); // case sensitive
+    }
+}
@@ -0,0 +1,111 @@
+#[macro_use]
+mod macros;
+pub mod ast;
+mod kinds;
+mod parser;
+#[cfg(test)]
+mod tests;
+mod token_set;
+mod tokenizer;
+
+use std::marker::PhantomData;
+
+use self::kinds::SYNTAX_KIND_MAX;
+pub use self::{kinds::ParseSyntaxKindError, kinds::SyntaxKind, tokenizer::tokenize};
+
+use ast::AstNode;
+pub use parser::ParseError;
+use rowan::GreenNode;
+pub use rowan::{NodeOrToken, TextRange, TextSize, TokenAtOffset, WalkEvent};
+pub(crate) use token_set::TokenSet;
+pub use tokenizer::Token;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum NixLanguage {}
+
+impl rowan::Language for NixLanguage {
+    type Kind = SyntaxKind;
+    fn kind_from_raw(raw: rowan::SyntaxKind) -> Self::Kind {
+        let discriminant: u16 = raw.0;
+        assert!(discriminant <= SYNTAX_KIND_MAX);
+        unsafe { std::mem::transmute::<u16, SyntaxKind>(discriminant) }
+    }
+    fn kind_to_raw(kind: Self::Kind) -> rowan::SyntaxKind {
+        rowan::SyntaxKind(kind as u16)
+    }
+}
+
+pub type SyntaxNode = rowan::SyntaxNode<NixLanguage>;
+pub type SyntaxToken = rowan::SyntaxToken<NixLanguage>;
+pub type SyntaxElement = rowan::NodeOrToken<SyntaxNode, SyntaxToken>;
+pub type SyntaxElementChildren = rowan::SyntaxElementChildren<NixLanguage>;
+pub type SyntaxNodeChildren = rowan::SyntaxNodeChildren<NixLanguage>;
+
+pub use ast::Root;
+
+impl Root {
+    pub fn parse(s: &str) -> Parse<Root> {
+        let (green, errors) = parser::parse(tokenize(s));
+        Parse { green, errors, _ty: PhantomData }
+    }
+}
+
+/// The result of a parse
+#[derive(Clone)]
+pub struct Parse<T> {
+    green: GreenNode,
+    errors: Vec<ParseError>,
+    _ty: PhantomData<fn() -> T>,
+}
+
+impl<T> Parse<T> {
+    pub fn syntax(&self) -> SyntaxNode {
+        SyntaxNode::new_root(self.green.clone())
+    }
+}
+
+impl<T: AstNode> Parse<T> {
+    pub fn tree(&self) -> T {
+        T::cast(self.syntax()).unwrap()
+    }
+
+    /// Return all errors in the tree, if any
+    pub fn errors(&self) -> &[ParseError] {
+        &self.errors
+    }
+
+    /// Either return the first error in the tree, or if there are none return self
+    pub fn ok(self) -> Result<T, ParseError> {
+        if let Some(err) = self.errors().first() {
+            return Err(err.clone());
+        }
+        Ok(self.tree())
+    }
+}
+
+/// Matches a `SyntaxNode` against an `ast` type.
+///
+/// # Example:
+///
+/// ```ignore
+/// match_ast! {
+///     match node {
+///         ast::Apply(it) => { ... },
+///         ast::Assert(it) => { ... },
+///         ast::IfElse(it) => { ... },
+///         _ => None,
+///     }
+/// }
+/// ```
+#[macro_export]
+macro_rules! match_ast {
+    (match $node:ident { $($tt:tt)* }) => { match_ast!(match ($node) { $($tt)* }) };
+
+    (match ($node:expr) {
+        $( ast::$ast:ident($it:pat) => $res:expr, )*
+        _ => $catch_all:expr $(,)?
+    }) => {{
+        $( if let Some($it) = ast::$ast::cast($node.clone()) { $res } else )*
+        { $catch_all }
+    }};
+}
@@ -0,0 +1,52 @@
+#[macro_export]
+#[rustfmt::skip]
+macro_rules! T {
+    (assert)   => ($crate::SyntaxKind::TOKEN_ASSERT);
+    (else)     => ($crate::SyntaxKind::TOKEN_ELSE);
+    (if)       => ($crate::SyntaxKind::TOKEN_IF);
+    (in)       => ($crate::SyntaxKind::TOKEN_IN);
+    (inherit)  => ($crate::SyntaxKind::TOKEN_INHERIT);
+    (let)      => ($crate::SyntaxKind::TOKEN_LET);
+    (or)       => ($crate::SyntaxKind::TOKEN_OR);
+    (rec)      => ($crate::SyntaxKind::TOKEN_REC);
+    (then)     => ($crate::SyntaxKind::TOKEN_THEN);
+    (with)     => ($crate::SyntaxKind::TOKEN_WITH);
+    (__curPos) => ($crate::SyntaxKind::TOKEN_CUR_POS);
+
+    ('{')     => ($crate::SyntaxKind::TOKEN_L_BRACE);
+    ('}')     => ($crate::SyntaxKind::TOKEN_R_BRACE);
+    ('[')     => ($crate::SyntaxKind::TOKEN_L_BRACK);
+    (']')     => ($crate::SyntaxKind::TOKEN_R_BRACK);
+    ('(')     => ($crate::SyntaxKind::TOKEN_L_PAREN);
+    (')')     => ($crate::SyntaxKind::TOKEN_R_PAREN);
+
+    (=)       => ($crate::SyntaxKind::TOKEN_ASSIGN);
+    (@)       => ($crate::SyntaxKind::TOKEN_AT);
+    (:)       => ($crate::SyntaxKind::TOKEN_COLON);
+    (,)       => ($crate::SyntaxKind::TOKEN_COMMA);
+    (.)       => ($crate::SyntaxKind::TOKEN_DOT);
+    (...)     => ($crate::SyntaxKind::TOKEN_ELLIPSIS);
+    (?)       => ($crate::SyntaxKind::TOKEN_QUESTION);
+    (;)       => ($crate::SyntaxKind::TOKEN_SEMICOLON);
+    (++)      => ($crate::SyntaxKind::TOKEN_CONCAT);
+    (!)       => ($crate::SyntaxKind::TOKEN_INVERT);
+    ("//")    => ($crate::SyntaxKind::TOKEN_UPDATE);
+
+    (+)       => ($crate::SyntaxKind::TOKEN_ADD);
+    (-)       => ($crate::SyntaxKind::TOKEN_SUB);
+    (*)       => ($crate::SyntaxKind::TOKEN_MUL);
+    (/)       => ($crate::SyntaxKind::TOKEN_DIV);
+
+    (&&)      => ($crate::SyntaxKind::TOKEN_AND_AND);
+    (==)      => ($crate::SyntaxKind::TOKEN_EQUAL);
+    (->)      => ($crate::SyntaxKind::TOKEN_IMPLICATION);
+    (<)       => ($crate::SyntaxKind::TOKEN_LESS);
+    (<=)      => ($crate::SyntaxKind::TOKEN_LESS_OR_EQ);
+    (>)       => ($crate::SyntaxKind::TOKEN_MORE);
+    (>=)      => ($crate::SyntaxKind::TOKEN_MORE_OR_EQ);
+    (!=)      => ($crate::SyntaxKind::TOKEN_NOT_EQUAL);
+    (||)      => ($crate::SyntaxKind::TOKEN_OR_OR);
+    ("|>")      => ($crate::SyntaxKind::TOKEN_PIPE_RIGHT);
+    ("<|")      => ($crate::SyntaxKind::TOKEN_PIPE_LEFT);
+    ($kind:ident) => ($crate::SyntaxKind::$kind);
+}
@@ -0,0 +1,834 @@
+//! The parser: turns a series of tokens into an AST
+
+use std::{collections::VecDeque, fmt};
+
+use rowan::{Checkpoint, GreenNode, GreenNodeBuilder, Language, TextRange, TextSize};
+
+use crate::{
+    tokenizer::Token,
+    NixLanguage,
+    SyntaxKind::{self, *},
+    TokenSet,
+};
+
+/// An error that occurred during parsing
+#[derive(Clone, Debug, PartialEq)]
+#[non_exhaustive]
+pub enum ParseError {
+    /// Unexpected is used when the cause cannot be specified further
+    Unexpected(TextRange),
+    /// UnexpectedExtra is used when there are additional tokens to the root in the tree
+    UnexpectedExtra(TextRange),
+    /// UnexpectedWanted is used when specific tokens are expected, but different one is found
+    UnexpectedWanted(SyntaxKind, TextRange, Box<[SyntaxKind]>),
+    /// UnexpectedDoubleBind is used when a pattern is bound twice
+    UnexpectedDoubleBind(TextRange),
+    /// UnexpectedEOF is used when the end of file is reached, while tokens are still expected
+    UnexpectedEOF,
+    /// UnexpectedEOFWanted is used when specific tokens are expected, but the end of file is reached
+    UnexpectedEOFWanted(Box<[SyntaxKind]>),
+    /// DuplicatedArgs is used when formal arguments are duplicated, e.g. `{ a, a }`
+    DuplicatedArgs(TextRange, String),
+    /// RecursionLimitExceeded is used when we're unable to parse further due to likely being close to
+    /// a stack overflow.
+    RecursionLimitExceeded,
+}
+
+impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ParseError::Unexpected(range) => {
+                write!(
+                    f,
+                    "error node at {}..{}",
+                    usize::from(range.start()),
+                    usize::from(range.end())
+                )
+            }
+            ParseError::UnexpectedExtra(range) => {
+                write!(
+                    f,
+                    "unexpected token at {}..{}",
+                    usize::from(range.start()),
+                    usize::from(range.end())
+                )
+            }
+            ParseError::UnexpectedWanted(got, range, kinds) => write!(
+                f,
+                "unexpected {:?} at {}..{}, wanted any of {:?}",
+                got,
+                usize::from(range.start()),
+                usize::from(range.end()),
+                kinds
+            ),
+            ParseError::UnexpectedDoubleBind(range) => {
+                write!(
+                    f,
+                    "unexpected double bind at {}..{}",
+                    usize::from(range.start()),
+                    usize::from(range.end())
+                )
+            }
+            ParseError::UnexpectedEOF => write!(f, "unexpected end of file"),
+            ParseError::UnexpectedEOFWanted(kinds) => {
+                write!(f, "unexpected end of file, wanted any of {:?}", kinds)
+            }
+            ParseError::DuplicatedArgs(range, ident) => {
+                write!(
+                    f,
+                    "argument `{}` is duplicated in {}..{}",
+                    ident,
+                    usize::from(range.start()),
+                    usize::from(range.end())
+                )
+            }
+            ParseError::RecursionLimitExceeded => write!(f, "recursion limit exceeded"),
+        }
+    }
+}
+
+impl std::error::Error for ParseError {}
+
+struct Parser<'a, I>
+where
+    I: Iterator<Item = Token<'a>>,
+{
+    builder: GreenNodeBuilder<'static>,
+    errors: Vec<ParseError>,
+
+    trivia_buffer: Vec<Token<'a>>,
+    buffer: VecDeque<Token<'a>>,
+    iter: I,
+    consumed: TextSize,
+
+    // Recursion depth, used for avoiding stack overflows. This may be incremented
+    // by any method as long as it is decremented when that method returns.
+    depth: u32,
+}
+impl<'a, I> Parser<'a, I>
+where
+    I: Iterator<Item = Token<'a>>,
+{
+    fn new(iter: I) -> Self {
+        Self {
+            builder: GreenNodeBuilder::new(),
+            errors: Vec::new(),
+
+            trivia_buffer: Vec::with_capacity(1),
+            buffer: VecDeque::with_capacity(1),
+            iter,
+            consumed: TextSize::from(0),
+
+            depth: 0,
+        }
+    }
+
+    fn get_text_position(&self) -> TextSize {
+        self.consumed
+    }
+
+    fn peek_raw(&mut self) -> Option<&Token<'a>> {
+        if self.buffer.is_empty() {
+            if let Some(token) = self.iter.next() {
+                self.buffer.push_back(token);
+            }
+        }
+        self.buffer.front()
+    }
+    fn drain_trivia_buffer(&mut self) {
+        for (t, s) in self.trivia_buffer.drain(..) {
+            self.consumed += TextSize::of(s);
+            self.builder.token(NixLanguage::kind_to_raw(t), s);
+        }
+    }
+    fn eat_trivia(&mut self) {
+        self.peek();
+        self.drain_trivia_buffer();
+    }
+    fn start_node(&mut self, kind: SyntaxKind) {
+        self.eat_trivia();
+        self.builder.start_node(NixLanguage::kind_to_raw(kind));
+    }
+    fn checkpoint(&mut self) -> Checkpoint {
+        self.eat_trivia();
+        self.builder.checkpoint()
+    }
+    fn start_node_at(&mut self, checkpoint: Checkpoint, kind: SyntaxKind) {
+        self.builder.start_node_at(checkpoint, NixLanguage::kind_to_raw(kind));
+    }
+    fn finish_node(&mut self) {
+        self.builder.finish_node();
+    }
+    fn start_error_node(&mut self) -> TextSize {
+        self.start_node(NODE_ERROR);
+        self.get_text_position()
+    }
+    fn finish_error_node(&mut self) -> TextSize {
+        self.finish_node();
+        self.get_text_position()
+    }
+    fn bump(&mut self) {
+        match self.try_next() {
+            Some((token, s)) => {
+                if token.is_trivia() {
+                    self.trivia_buffer.push((token, s))
+                } else {
+                    self.drain_trivia_buffer();
+                    self.manual_bump(s, token);
+                }
+            }
+            None => self.errors.push(ParseError::UnexpectedEOF),
+        }
+    }
+    fn try_next(&mut self) -> Option<Token<'a>> {
+        self.buffer.pop_front().or_else(|| self.iter.next())
+    }
+    fn manual_bump(&mut self, s: &str, token: SyntaxKind) {
+        self.consumed += TextSize::of(s);
+        self.builder.token(NixLanguage::kind_to_raw(token), s)
+    }
+    fn peek(&mut self) -> Option<SyntaxKind> {
+        while self.peek_raw().map(|&(t, _)| t.is_trivia()).unwrap_or(false) {
+            self.bump();
+        }
+        self.peek_ahead(0)
+    }
+    fn peek_ahead(&mut self, n: usize) -> Option<SyntaxKind> {
+        let mut non_trivia_seen = 0;
+        let mut cursor = 0;
+        loop {
+            if self.buffer.len() <= cursor {
+                self.buffer.push_back(self.iter.next()?);
+            }
+            let kind = self.buffer[cursor].0;
+            cursor += 1;
+            if !kind.is_trivia() {
+                if non_trivia_seen == n {
+                    return Some(kind);
+                }
+                non_trivia_seen += 1;
+            }
+        }
+    }
+    fn expect_peek_any(&mut self, allowed_slice: &[SyntaxKind]) -> Option<SyntaxKind> {
+        let allowed = TokenSet::from_slice(allowed_slice);
+
+        let next = match self.peek() {
+            None => None,
+            Some(kind) if allowed.contains(kind) => Some(kind),
+            Some(kind) => {
+                let start = self.start_error_node();
+                loop {
+                    self.bump();
+                    if self.peek().map(|kind| allowed.contains(kind)).unwrap_or(true) {
+                        break;
+                    }
+                }
+                let end = self.finish_error_node();
+                self.errors.push(ParseError::UnexpectedWanted(
+                    kind,
+                    TextRange::new(start, end),
+                    allowed_slice.to_vec().into_boxed_slice(),
+                ));
+
+                self.peek()
+            }
+        };
+        if next.is_none() {
+            self.errors
+                .push(ParseError::UnexpectedEOFWanted(allowed_slice.to_vec().into_boxed_slice()));
+        }
+        next
+    }
+    fn expect(&mut self, expected: SyntaxKind) {
+        if self.expect_peek_any(&[expected]).is_some() {
+            self.bump();
+        }
+    }
+
+    fn expect_ident(&mut self) {
+        if self.expect_peek_any(&[TOKEN_IDENT]).is_some() {
+            self.start_node(NODE_IDENT);
+            self.bump();
+            self.finish_node()
+        }
+    }
+
+    fn parse_dynamic(&mut self) {
+        self.start_node(NODE_DYNAMIC);
+        self.bump();
+        while self.peek().map(|t| t != TOKEN_INTERPOL_END).unwrap_or(false) {
+            self.parse_expr();
+        }
+        self.bump();
+        self.finish_node();
+    }
+
+    fn parse_string(&mut self) {
+        self.start_node(NODE_STRING);
+        self.expect(TOKEN_STRING_START);
+
+        loop {
+            match self.expect_peek_any(&[
+                TOKEN_STRING_END,
+                TOKEN_STRING_CONTENT,
+                TOKEN_INTERPOL_START,
+            ]) {
+                Some(TOKEN_STRING_CONTENT) => self.bump(),
+                Some(TOKEN_INTERPOL_START) => {
+                    self.start_node(NODE_INTERPOL);
+                    self.bump();
+                    self.parse_expr();
+                    self.expect(TOKEN_INTERPOL_END);
+                    self.finish_node();
+                }
+                // handled by expect_peek_any
+                _ => break,
+            }
+        }
+        self.expect(TOKEN_STRING_END);
+
+        self.finish_node();
+    }
+    fn parse_attr(&mut self) {
+        match self.peek() {
+            Some(TOKEN_INTERPOL_START) => self.parse_dynamic(),
+            Some(TOKEN_STRING_START) => self.parse_string(),
+            _ => {
+                // NB: the tokenizer always emits TOKEN_IDENT for __curPos;
+                // TOKEN_CUR_POS is included for completeness but cannot
+                // currently be produced by the tokenizer.
+                if self.expect_peek_any(&[TOKEN_IDENT, TOKEN_OR, TOKEN_CUR_POS]).is_some() {
+                    self.start_node(NODE_IDENT);
+                    let (_, s) = self.try_next().unwrap();
+                    self.manual_bump(s, TOKEN_IDENT);
+                    self.finish_node()
+                }
+            }
+        }
+    }
+    fn parse_attrpath(&mut self) {
+        self.start_node(NODE_ATTRPATH);
+        loop {
+            self.parse_attr();
+
+            if self.peek() == Some(T![.]) {
+                self.bump();
+            } else {
+                break;
+            }
+        }
+        self.finish_node();
+    }
+    fn parse_pattern(&mut self, bound: bool) {
+        if self.peek().map(|t| t == T!['}']).unwrap_or(true) {
+            self.bump();
+        } else {
+            loop {
+                match self.expect_peek_any(&[T!['}'], T![...], TOKEN_IDENT]) {
+                    Some(T!['}']) => {
+                        self.bump();
+                        break;
+                    }
+                    Some(T![...]) => {
+                        self.bump();
+                        self.expect(T!['}']);
+                        break;
+                    }
+                    Some(TOKEN_IDENT) => {
+                        self.start_node(NODE_PAT_ENTRY);
+                        self.expect_ident();
+                        if let Some(T![?]) = self.peek() {
+                            self.bump();
+                            self.parse_expr();
+                        }
+                        self.finish_node();
+                        match self.peek() {
+                            Some(T![,]) => self.bump(),
+                            _ => {
+                                self.expect(T!['}']);
+                                break;
+                            }
+                        }
+                    }
+                    // handled by expect_peek_any
+                    _ => break,
+                }
+            }
+        }
+
+        if self.peek() == Some(T![@]) {
+            let kind = if bound { NODE_ERROR } else { NODE_PAT_BIND };
+            self.start_node(kind);
+            let start = self.get_text_position();
+            self.bump();
+            self.expect_ident();
+            let end = self.finish_error_node();
+            if bound {
+                self.errors.push(ParseError::UnexpectedDoubleBind(TextRange::new(start, end)));
+            }
+        }
+    }
+    fn parse_set(&mut self, until: SyntaxKind) {
+        loop {
+            match self.peek() {
+                None => break,
+                token if token == Some(until) => break,
+                Some(T![inherit]) => {
+                    self.start_node(NODE_INHERIT);
+                    self.bump();
+
+                    if self.peek() == Some(T!['(']) {
+                        self.start_node(NODE_INHERIT_FROM);
+                        self.bump();
+                        self.parse_expr();
+                        self.expect(T![')']);
+                        self.finish_node();
+                    }
+
+                    loop {
+                        match self.peek() {
+                            Some(t) if t != T![;] => {
+                                self.parse_attr();
+                            }
+                            Some(_) => {
+                                break;
+                            }
+                            None => {
+                                self.errors.push(ParseError::UnexpectedEOF);
+                                break;
+                            }
+                        }
+                    }
+
+                    self.expect(T![;]);
+                    self.finish_node();
+                }
+                Some(_) => {
+                    self.start_node(NODE_ATTRPATH_VALUE);
+                    self.parse_attrpath();
+                    self.expect(T![=]);
+                    self.parse_expr();
+                    self.expect(T![;]);
+                    self.finish_node();
+                }
+            }
+        }
+        self.bump(); // the final close, like '}'
+    }
+
+    fn parse_simple(&mut self) -> Checkpoint {
+        let peek = match self.peek() {
+            Some(it) => it,
+            None => {
+                self.errors.push(ParseError::UnexpectedEOF);
+                // NB: we don't use `self.checkpoint()` here in order to avoid
+                // eating the whitespace. The actual checkpoint doesn't matter
+                // in this case and, ideally, should be returning `None`, but
+                // that makes code slightly more complex for little real
+                // benefit.
+                return self.builder.checkpoint();
+            }
+        };
+        let checkpoint = self.checkpoint();
+        match peek {
+            T!['('] => {
+                self.start_node(NODE_PAREN);
+                self.bump();
+                self.parse_expr();
+                self.bump();
+                self.finish_node();
+            }
+            T![rec] => {
+                self.start_node(NODE_ATTR_SET);
+                self.bump();
+                self.expect(T!['{']);
+                self.parse_set(T!['}']);
+                self.finish_node();
+            }
+            T!['{'] => {
+                // Do a lookahead:
+                let mut peek = [None, None];
+                for i in &mut peek {
+                    let mut token;
+                    *i = loop {
+                        token = self.iter.next();
+                        let kind = token.as_ref().map(|&(t, _)| t);
+                        if let Some(token) = token {
+                            self.buffer.push_back(token);
+                        }
+                        if kind.map(|t| !t.is_trivia()).unwrap_or(true) {
+                            break kind;
+                        }
+                    };
+                }
+
+                match peek {
+                    [Some(TOKEN_IDENT), Some(T![,])]
+                    | [Some(TOKEN_IDENT), Some(T![?])]
+                    | [Some(TOKEN_IDENT), Some(T!['}'])]
+                    | [Some(T![...]), Some(T!['}'])]
+                    | [Some(T!['}']), Some(T![:])]
+                    | [Some(T!['}']), Some(T![@])] => {
+                        // This looks like a pattern
+                        self.start_node(NODE_LAMBDA);
+
+                        self.start_node(NODE_PATTERN);
+                        self.bump();
+                        self.parse_pattern(false);
+                        self.finish_node();
+
+                        self.expect(T![:]);
+                        self.parse_expr();
+
+                        self.finish_node();
+                    }
+                    _ => {
+                        // This looks like a set
+                        self.start_node(NODE_ATTR_SET);
+                        self.bump();
+                        self.parse_set(T!['}']);
+                        self.finish_node();
+                    }
+                }
+            }
+            T!['['] => {
+                self.start_node(NODE_LIST);
+                self.bump();
+                while self.peek().map(|t| t != T![']']).unwrap_or(false) {
+                    self.parse_simple();
+                }
+                self.bump();
+                self.finish_node();
+            }
+            TOKEN_STRING_START => self.parse_string(),
+            TOKEN_PATH_ABS | TOKEN_PATH_REL | TOKEN_PATH_HOME | TOKEN_PATH_SEARCH => {
+                let node_kind = match self.peek().unwrap() {
+                    TOKEN_PATH_ABS => NODE_PATH_ABS,
+                    TOKEN_PATH_REL => NODE_PATH_REL,
+                    TOKEN_PATH_HOME => NODE_PATH_HOME,
+                    TOKEN_PATH_SEARCH => NODE_PATH_SEARCH,
+                    _ => unreachable!(),
+                };
+                self.start_node(node_kind);
+                self.bump();
+
+                // Search paths (<nixpkgs>) don't support interpolation
+                let is_search_path = node_kind == NODE_PATH_SEARCH;
+                let is_complex_path = !is_search_path && self.peek() == Some(TOKEN_INTERPOL_START);
+
+                if is_complex_path {
+                    loop {
+                        match self.peek_raw().map(|(t, _)| t) {
+                            Some(TOKEN_PATH_ABS) | Some(TOKEN_PATH_REL) | Some(TOKEN_PATH_HOME) => {
+                                self.bump()
+                            }
+                            Some(TOKEN_INTERPOL_START) => {
+                                self.start_node(NODE_INTERPOL);
+                                self.bump();
+                                self.parse_expr();
+                                self.expect(TOKEN_INTERPOL_END);
+                                self.finish_node();
+                            }
+                            _ => break,
+                        }
+                    }
+                }
+                self.finish_node();
+            }
+            t if t.is_literal() => {
+                self.start_node(NODE_LITERAL);
+                self.bump();
+                self.finish_node();
+            }
+            TOKEN_IDENT => {
+                let ident = self.peek_raw().map(|&(_, s)| s);
+
+                match self.peek_ahead(1) {
+                    Some(T![:]) => {
+                        self.expect_ident();
+
+                        self.start_node_at(checkpoint, NODE_LAMBDA);
+                        self.start_node_at(checkpoint, NODE_IDENT_PARAM);
+                        self.finish_node();
+                        self.expect(T![:]);
+                        self.parse_expr();
+                        self.finish_node();
+                    }
+                    Some(T![@]) => {
+                        self.expect_ident();
+
+                        self.start_node_at(checkpoint, NODE_LAMBDA);
+                        self.start_node_at(checkpoint, NODE_PATTERN);
+                        self.start_node_at(checkpoint, NODE_PAT_BIND);
+                        self.expect(T![@]);
+                        self.finish_node(); // PatBind
+
+                        self.expect(T!['{']);
+                        self.parse_pattern(true);
+                        self.finish_node(); // Pattern
+
+                        self.expect(T![:]);
+                        self.parse_expr();
+                        self.finish_node(); // Lambda
+                    }
+                    _ if ident == Some("__curPos") => {
+                        self.start_node(NODE_CUR_POS);
+                        let (_, s) = self.try_next().unwrap();
+                        self.manual_bump(s, TOKEN_CUR_POS);
+                        self.finish_node();
+                    }
+                    _ => self.expect_ident(),
+                }
+            }
+            kind => {
+                let start = self.start_error_node();
+                self.bump();
+                let end = self.finish_error_node();
+                self.errors.push(ParseError::UnexpectedWanted(
+                    kind,
+                    TextRange::new(start, end),
+                    [T!['('], T![rec], T!['{'], T!['['], TOKEN_STRING_START, TOKEN_IDENT]
+                        .to_vec()
+                        .into_boxed_slice(),
+                ));
+            }
+        };
+
+        if self.peek() == Some(T![.]) {
+            self.start_node_at(checkpoint, NODE_SELECT);
+            self.bump();
+            self.parse_attrpath();
+            if self.peek() == Some(T![or]) {
+                self.bump();
+                self.parse_simple();
+            }
+            self.finish_node();
+
+        // This seems weird, but it matches Nix's behavior.
+        // If there is no "." but a "or" immediately followed a primary expression,
+        // we construct a application node, with "or" parsed as an identifier,
+        // ignoring the associativity of ancestor applications.
+        // Eg.
+        // "a b or c" => "((a (b or)) c)"
+        // "or a" => fail
+        } else if self.peek() == Some(T![or]) {
+            self.start_node_at(checkpoint, NODE_APPLY);
+            self.start_node(NODE_IDENT);
+            let (_, s) = self.try_next().unwrap();
+            self.manual_bump(s, TOKEN_IDENT);
+            self.finish_node();
+            self.finish_node();
+        }
+
+        checkpoint
+    }
+    fn parse_fn(&mut self) -> Checkpoint {
+        let checkpoint = self.parse_simple();
+
+        while self.peek().map(|t| t.is_fn_arg()).unwrap_or(false) {
+            self.start_node_at(checkpoint, NODE_APPLY);
+            self.parse_simple();
+            self.finish_node();
+        }
+        checkpoint
+    }
+    fn parse_negate(&mut self) -> Checkpoint {
+        if self.peek() == Some(T![-]) {
+            let checkpoint = self.checkpoint();
+            self.start_node(NODE_UNARY_OP);
+            self.bump();
+            self.parse_negate();
+            self.finish_node();
+            checkpoint
+        } else {
+            self.parse_fn()
+        }
+    }
+    fn parse_non_assoc(&mut self, next: fn(&mut Self) -> Checkpoint, ops: TokenSet) -> Checkpoint {
+        let checkpoint = next(self);
+        if self.peek().map(|t| ops.contains(t)).unwrap_or(false) {
+            self.start_node_at(checkpoint, NODE_BIN_OP);
+            self.bump();
+            next(self);
+            self.finish_node();
+        }
+        checkpoint
+    }
+    fn parse_left_assoc(&mut self, next: fn(&mut Self) -> Checkpoint, ops: TokenSet) -> Checkpoint {
+        let checkpoint = next(self);
+        while self.peek().map(|t| ops.contains(t)).unwrap_or(false) {
+            self.start_node_at(checkpoint, NODE_BIN_OP);
+            self.bump();
+            next(self);
+            self.finish_node();
+        }
+        checkpoint
+    }
+    fn parse_right_assoc(
+        &mut self,
+        next: fn(&mut Self) -> Checkpoint,
+        ops: TokenSet,
+    ) -> Checkpoint {
+        let checkpoint = next(self);
+        if self.peek().map(|t| ops.contains(t)).unwrap_or(false) {
+            self.start_node_at(checkpoint, NODE_BIN_OP);
+            self.bump();
+            self.parse_right_assoc(next, ops);
+            self.finish_node();
+        }
+        checkpoint
+    }
+    fn parse_hasattr(&mut self) -> Checkpoint {
+        let checkpoint = self.parse_negate();
+        while self.peek().map(|t| t == T![?]).unwrap_or(false) {
+            self.start_node_at(checkpoint, NODE_HAS_ATTR);
+            self.bump();
+            self.parse_attrpath();
+            self.finish_node();
+        }
+        checkpoint
+    }
+    fn parse_concat(&mut self) -> Checkpoint {
+        self.parse_right_assoc(Self::parse_hasattr, T![++] | ())
+    }
+    fn parse_mul(&mut self) -> Checkpoint {
+        self.parse_left_assoc(Self::parse_concat, T![*] | T![/])
+    }
+    fn parse_add(&mut self) -> Checkpoint {
+        self.parse_left_assoc(Self::parse_mul, T![+] | T![-])
+    }
+    fn parse_invert(&mut self) -> Checkpoint {
+        if self.peek() == Some(TOKEN_INVERT) {
+            let checkpoint = self.checkpoint();
+            self.start_node(NODE_UNARY_OP);
+            self.bump();
+            self.parse_invert();
+            self.finish_node();
+            checkpoint
+        } else {
+            self.parse_add()
+        }
+    }
+    fn parse_merge(&mut self) -> Checkpoint {
+        self.parse_right_assoc(Self::parse_invert, T!["//"] | ())
+    }
+    fn parse_compare(&mut self) -> Checkpoint {
+        self.parse_non_assoc(Self::parse_merge, T![<] | T![<=] | T![>] | T![>=])
+    }
+    fn parse_equal(&mut self) -> Checkpoint {
+        self.parse_non_assoc(Self::parse_compare, T![==] | T![!=])
+    }
+    fn parse_and(&mut self) -> Checkpoint {
+        self.parse_left_assoc(Self::parse_equal, T![&&] | ())
+    }
+    fn parse_or(&mut self) -> Checkpoint {
+        self.parse_left_assoc(Self::parse_and, T![||] | ())
+    }
+    fn parse_implication(&mut self) -> Checkpoint {
+        self.parse_right_assoc(Self::parse_or, T![->] | ())
+    }
+    #[inline(always)]
+    fn parse_math(&mut self) -> Checkpoint {
+        // Always point this to the lowest-level math function there is
+        self.parse_implication()
+    }
+    fn parse_pipe_right(&mut self) -> Checkpoint {
+        self.parse_left_assoc(Self::parse_math, T!["|>"] | ())
+    }
+    fn parse_pipe_left(&mut self) -> Checkpoint {
+        self.parse_right_assoc(Self::parse_pipe_right, T!["<|"] | ())
+    }
+    /// Parse Nix code into an AST
+    pub fn parse_expr(&mut self) -> Checkpoint {
+        // Limit chosen somewhat arbitrarily
+        if self.depth >= 512 {
+            self.errors.push(ParseError::RecursionLimitExceeded);
+            // Consume tokens to the end of the file. Erroring without bumping might cause
+            // infinite looping elsewhere.
+            self.start_error_node();
+            while self.peek().is_some() {
+                self.bump()
+            }
+            self.finish_error_node();
+            return self.checkpoint();
+        }
+        self.depth += 1;
+        let out = match self.peek() {
+            Some(T![let]) => {
+                let checkpoint = self.checkpoint();
+                self.bump();
+
+                if self.peek() == Some(T!['{']) {
+                    self.start_node_at(checkpoint, NODE_LEGACY_LET);
+                    self.bump();
+                    self.parse_set(T!['}']);
+                    self.finish_node();
+                } else {
+                    self.start_node_at(checkpoint, NODE_LET_IN);
+                    self.parse_set(T![in]);
+                    self.parse_expr();
+                    self.finish_node();
+                }
+                checkpoint
+            }
+            Some(T![with]) => {
+                let checkpoint = self.checkpoint();
+                self.start_node(NODE_WITH);
+                self.bump();
+                self.parse_expr();
+                self.expect(T![;]);
+                self.parse_expr();
+                self.finish_node();
+                checkpoint
+            }
+            Some(T![if]) => {
+                let checkpoint = self.checkpoint();
+                self.start_node(NODE_IF_ELSE);
+                self.bump();
+                self.parse_expr();
+                self.expect(T![then]);
+                self.parse_expr();
+                self.expect(TOKEN_ELSE);
+                self.parse_expr();
+                self.finish_node();
+                checkpoint
+            }
+            Some(T![assert]) => {
+                let checkpoint = self.checkpoint();
+                self.start_node(NODE_ASSERT);
+                self.bump();
+                self.parse_expr();
+                self.expect(T![;]);
+                self.parse_expr();
+                self.finish_node();
+                checkpoint
+            }
+            _ => self.parse_pipe_left(),
+        };
+        self.depth -= 1;
+        out
+    }
+}
+
+/// Parse tokens into an AST
+pub fn parse<'s, I>(iter: I) -> (GreenNode, Vec<ParseError>)
+where
+    I: Iterator<Item = Token<'s>>,
+{
+    let mut parser = Parser::new(iter);
+    parser.builder.start_node(NixLanguage::kind_to_raw(NODE_ROOT));
+    parser.parse_expr();
+    parser.eat_trivia();
+    if parser.peek().is_some() {
+        let start = parser.start_error_node();
+        while parser.peek().is_some() {
+            parser.bump();
+        }
+        let end = parser.finish_error_node();
+        parser.errors.push(ParseError::UnexpectedExtra(TextRange::new(start, end)));
+        parser.eat_trivia();
+    }
+    parser.builder.finish_node();
+    (parser.builder.finish(), parser.errors)
+}
@@ -0,0 +1,127 @@
+use std::{ffi::OsStr, fmt::Write, fs, path::PathBuf};
+
+use expect_test::expect_file;
+use rowan::ast::AstNode;
+
+use crate::{
+    ast::{self, HasEntry},
+    tokenize, Root, SyntaxKind,
+};
+
+#[test]
+fn interpolation() {
+    let root = ast::Root::parse(include_str!("../test_data/parser/success/interpolation.nix"))
+        .ok()
+        .unwrap();
+    let let_in = ast::LetIn::try_from(root.expr().unwrap()).unwrap();
+    let set = ast::AttrSet::try_from(let_in.body().unwrap()).unwrap();
+    let entry = set.entries().nth(1).unwrap();
+    let attrpath_value = ast::AttrpathValue::try_from(entry).unwrap();
+    let value = ast::Str::try_from(attrpath_value.value().unwrap()).unwrap();
+
+    match &*value.normalized_parts() {
+    &[
+        ast::InterpolPart::Literal(ref s1),
+        ast::InterpolPart::Interpolation(_),
+        ast::InterpolPart::Literal(ref s2),
+        ast::InterpolPart::Interpolation(_),
+        ast::InterpolPart::Literal(ref s3)
+    ]
+    if s1 == "The set's x value is: "
+        && s2 == "\n\nThis line shall have no indention\n  This line shall be indented by 2\n\n\n"
+        && s3 == "\n" => (),
+    parts => panic!("did not match: {:#?}", parts)
+}
+}
+
+#[test]
+fn inherit() {
+    let root =
+        ast::Root::parse(include_str!("../test_data/parser/success/inherit.nix")).ok().unwrap();
+    let let_in = ast::LetIn::try_from(root.expr().unwrap()).unwrap();
+    let set = ast::AttrSet::try_from(let_in.body().unwrap()).unwrap();
+    let inherit = set.inherits().nth(1).unwrap();
+
+    let from = inherit.from().unwrap().expr().unwrap();
+    let ident: ast::Ident = ast::Ident::try_from(from).unwrap();
+    assert_eq!(ident.syntax().text(), "set");
+    let mut children = inherit.attrs();
+    assert_eq!(children.next().unwrap().syntax().text(), "z");
+    assert_eq!(children.next().unwrap().syntax().text(), "a");
+    assert!(children.next().is_none());
+}
+
+#[test]
+fn math() {
+    let root = ast::Root::parse(include_str!("../test_data/parser/success/math.nix")).ok().unwrap();
+    let op1 = ast::BinOp::try_from(root.expr().unwrap()).unwrap();
+    let op2 = ast::BinOp::try_from(op1.lhs().unwrap()).unwrap();
+    assert_eq!(op1.operator().unwrap(), ast::BinOpKind::Add);
+
+    let lhs = ast::Literal::try_from(op2.lhs().unwrap()).unwrap();
+    assert_eq!(lhs.syntax().text(), "1");
+
+    let rhs = ast::BinOp::try_from(op2.rhs().unwrap()).unwrap();
+    assert_eq!(rhs.operator().unwrap(), ast::BinOpKind::Mul);
+}
+
+#[test]
+fn t_macro() {
+    assert_eq!(T![@], SyntaxKind::TOKEN_AT);
+    assert!(matches!(SyntaxKind::TOKEN_L_PAREN, T!['(']));
+}
+
+fn dir_tests<F>(dir: &str, get_actual: F)
+where
+    F: Fn(String) -> String,
+{
+    let base_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "test_data", dir].iter().collect();
+    let success_path = base_path.join("success");
+    let error_path = base_path.join("error");
+
+    let entries = success_path.read_dir().unwrap().chain(error_path.read_dir().unwrap());
+
+    for entry in entries {
+        let path = entry.unwrap().path();
+
+        if path.extension() != Some(OsStr::new("nix")) {
+            continue;
+        }
+
+        println!("testing: {}", path.display());
+
+        let mut code = fs::read_to_string(&path).unwrap();
+        if code.ends_with('\n') {
+            code.truncate(code.len() - 1);
+        }
+
+        let actual = get_actual(code);
+        expect_file![path.with_extension("expect")].assert_eq(&actual);
+    }
+}
+
+#[test]
+fn parser_dir_tests() {
+    dir_tests("parser", |code| {
+        let parse = Root::parse(&code);
+
+        let mut actual = String::new();
+        for error in parse.errors() {
+            writeln!(actual, "error: {}", error).unwrap();
+        }
+        writeln!(actual, "{:#?}", parse.syntax()).unwrap();
+
+        actual
+    })
+}
+
+#[test]
+fn tokenizer_dir_tests() {
+    dir_tests("tokenizer", |code| {
+        let mut actual = String::new();
+        for (kind, str) in tokenize(&code) {
+            writeln!(actual, "{:?}, \"{}\"", kind, str).unwrap();
+        }
+        actual
+    })
+}
@@ -0,0 +1,86 @@
+use std::ops;
+
+use crate::SyntaxKind;
+
+/// A bit-set of `SyntaxKind`s
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TokenSet(u128);
+
+impl TokenSet {
+    #[allow(dead_code)]
+    pub(crate) const EMPTY: TokenSet = TokenSet(0);
+
+    pub(crate) const fn new(kind: SyntaxKind) -> TokenSet {
+        TokenSet(mask(kind))
+    }
+
+    pub(crate) const fn from_slice(kinds: &[SyntaxKind]) -> TokenSet {
+        let mut res = 0u128;
+        let mut i = 0;
+        while i < kinds.len() {
+            res |= mask(kinds[i]);
+            i += 1
+        }
+        TokenSet(res)
+    }
+
+    pub(crate) const fn union(self, other: TokenSet) -> TokenSet {
+        TokenSet(self.0 | other.0)
+    }
+
+    pub(crate) const fn contains(&self, kind: SyntaxKind) -> bool {
+        self.0 & mask(kind) != 0
+    }
+}
+
+const fn mask(kind: SyntaxKind) -> u128 {
+    1u128 << (kind as usize)
+}
+
+impl ops::BitOr for SyntaxKind {
+    type Output = TokenSet;
+
+    fn bitor(self, rhs: Self) -> Self::Output {
+        TokenSet(mask(self) | mask(rhs))
+    }
+}
+
+impl ops::BitOr<SyntaxKind> for TokenSet {
+    type Output = TokenSet;
+
+    fn bitor(self, rhs: SyntaxKind) -> Self::Output {
+        self.union(TokenSet(mask(rhs)))
+    }
+}
+
+impl ops::BitOr<TokenSet> for SyntaxKind {
+    type Output = TokenSet;
+
+    fn bitor(self, rhs: TokenSet) -> Self::Output {
+        TokenSet(mask(self)).union(rhs)
+    }
+}
+
+impl ops::BitOr<TokenSet> for TokenSet {
+    type Output = TokenSet;
+
+    fn bitor(self, rhs: TokenSet) -> Self::Output {
+        self.union(rhs)
+    }
+}
+
+impl ops::BitOr<SyntaxKind> for () {
+    type Output = TokenSet;
+
+    fn bitor(self, rhs: SyntaxKind) -> Self::Output {
+        TokenSet::new(rhs)
+    }
+}
+
+impl ops::BitOr<()> for SyntaxKind {
+    type Output = TokenSet;
+
+    fn bitor(self, (): ()) -> Self::Output {
+        TokenSet::new(self)
+    }
+}
@@ -0,0 +1,566 @@
+//! The tokenizer: turns a string into tokens, such as numbers, strings, and keywords
+
+use crate::SyntaxKind::{self, *};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum IdentType {
+    Ident,
+    PathRel,
+    PathAbs,
+    PathHome,
+    PathSearch,
+    Uri,
+}
+
+fn is_valid_path_char(c: char) -> bool {
+    matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '/' | '_' | '.' | '+' | '-')
+}
+
+fn is_valid_uri_char(c: char) -> bool {
+    match c {
+        '%' | '?' | ':' | '@' | '&' | '=' | '$' | ',' | '!' | '~' | '*' | '\'' => true,
+        _ => is_valid_path_char(c),
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Context {
+    StringBody { multiline: bool },
+    StringEnd,
+    Interpol { brackets: u32 },
+    InterpolStart,
+    Path(SyntaxKind),
+}
+
+#[derive(Clone, Copy)]
+struct State<'a> {
+    input: &'a str,
+    offset: usize,
+}
+
+impl PartialEq for State<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        std::ptr::eq(self.input, other.input) && self.offset == other.offset
+    }
+}
+
+impl Eq for State<'_> {}
+
+pub type Token<'a> = (SyntaxKind, &'a str);
+
+/// Tokenize the given input
+pub fn tokenize(input: &str) -> impl Iterator<Item = Token<'_>> + '_ {
+    Tokenizer::new(input)
+}
+
+struct Tokenizer<'a> {
+    ctx: Vec<Context>,
+    state: State<'a>,
+}
+
+impl<'a> Tokenizer<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Self { ctx: Vec::new(), state: State { input, offset: 0 } }
+    }
+}
+
+impl Tokenizer<'_> {
+    fn remaining(&self) -> &str {
+        &self.state.input[self.state.offset..]
+    }
+    fn peek(&self) -> Option<char> {
+        self.remaining().chars().next()
+    }
+    fn next(&mut self) -> Option<char> {
+        let c = self.peek();
+        if let Some(c) = c {
+            self.state.offset += c.len_utf8();
+        }
+        c
+    }
+    fn starts_with_bump(&mut self, s: &str) -> bool {
+        let starts_with = self.remaining().starts_with(s);
+        if starts_with {
+            self.state.offset += s.len();
+        }
+        starts_with
+    }
+    fn str_since<'a>(&self, past: State<'a>) -> &'a str {
+        &past.input[past.offset..self.state.offset]
+    }
+
+    fn push_ctx(&mut self, ctx: Context) {
+        self.ctx.push(ctx)
+    }
+
+    fn pop_ctx(&mut self, ctx: Context) {
+        debug_assert!(self
+            .ctx
+            .last()
+            .is_some_and(|c| std::mem::discriminant(c) == std::mem::discriminant(&ctx)));
+        self.ctx.pop();
+    }
+
+    /// Consumes all next characters.
+    /// while the predicate function [f] returns true
+    ///
+    /// returns the count of consumed characters.
+    fn consume<F>(&mut self, mut f: F) -> usize
+    where
+        F: FnMut(char) -> bool,
+    {
+        let len: usize =
+            self.remaining().chars().take_while(|&c| f(c)).map(|c| c.len_utf8()).sum::<usize>();
+        self.state.offset += len;
+        len
+    }
+    fn next_string(&mut self, multiline: bool) -> SyntaxKind {
+        loop {
+            let start = self.state;
+            match self.next() {
+                None => {
+                    self.pop_ctx(Context::StringBody { multiline });
+                    return TOKEN_ERROR;
+                }
+                Some('"') if !multiline => {
+                    self.state = start;
+                    self.pop_ctx(Context::StringBody { multiline: false });
+                    self.push_ctx(Context::StringEnd);
+                    return TOKEN_STRING_CONTENT;
+                }
+                Some('\\') if !multiline => {
+                    if self.next().is_none() {
+                        return TOKEN_ERROR;
+                    }
+                }
+
+                Some('\'') if multiline => match self.peek() {
+                    None => return TOKEN_ERROR,
+                    Some('\'') => {
+                        self.next();
+                        match self.peek() {
+                            Some('\'') | Some('$') => {
+                                self.next().unwrap();
+                            }
+                            Some('\\') => {
+                                self.next().unwrap();
+                                if self.next().is_none() {
+                                    return TOKEN_ERROR;
+                                }
+                            }
+                            _ => {
+                                self.state = start;
+                                self.pop_ctx(Context::StringBody { multiline: true });
+                                self.push_ctx(Context::StringEnd);
+                                return TOKEN_STRING_CONTENT;
+                            }
+                        }
+                    }
+                    Some(_) => (),
+                },
+
+                Some('$') => match self.peek() {
+                    Some('$') => {
+                        self.next().unwrap();
+                    }
+                    Some('{') => {
+                        self.state = start;
+                        self.push_ctx(Context::InterpolStart);
+                        return TOKEN_STRING_CONTENT;
+                    }
+                    _ => (),
+                },
+                Some(_) => (),
+            }
+        }
+    }
+
+    fn check_path_since(&mut self, past: State, kind: SyntaxKind) -> SyntaxKind {
+        self.consume(is_valid_path_char);
+        let path_str = self.str_since(past);
+        if self.remaining().starts_with("${") {
+            self.ctx.push(Context::InterpolStart);
+        } else if path_str.ends_with('/') || path_str.contains("//") {
+            return TOKEN_ERROR;
+        } else {
+            self.pop_ctx(Context::Path(kind));
+        }
+        kind
+    }
+
+    fn next_inner(&mut self) -> Option<SyntaxKind> {
+        let start = self.state;
+
+        // Handle already started multi-token
+        loop {
+            match self.ctx.last() {
+                Some(Context::InterpolStart) => {
+                    self.pop_ctx(Context::InterpolStart);
+                    self.ctx.push(Context::Interpol { brackets: 0 });
+                    if self.starts_with_bump("${") {
+                        return Some(TOKEN_INTERPOL_START);
+                    } else {
+                        unreachable!()
+                    }
+                }
+                Some(Context::Path(path_kind)) => {
+                    let path_kind = *path_kind;
+
+                    // Search paths (<nixpkgs>) don't support interpolation
+                    if path_kind == TOKEN_PATH_SEARCH {
+                        if self.peek().is_some_and(is_valid_path_char) {
+                            return Some(self.check_path_since(start, path_kind));
+                        } else {
+                            self.pop_ctx(Context::Path(path_kind));
+                        }
+                    } else if self.starts_with_bump("${") {
+                        self.ctx.push(Context::Interpol { brackets: 0 });
+                        return Some(TOKEN_INTERPOL_START);
+                    } else if self.peek().is_some_and(is_valid_path_char) {
+                        return Some(self.check_path_since(start, path_kind));
+                    } else {
+                        self.pop_ctx(Context::Path(path_kind));
+                    }
+                }
+                Some(&Context::StringBody { multiline }) => {
+                    let token = self.next_string(multiline);
+                    // skip empty stuff
+                    if self.state == start {
+                        continue;
+                    }
+                    return Some(token);
+                }
+                Some(Context::StringEnd) => {
+                    self.pop_ctx(Context::StringEnd);
+                    let status = match self.peek() {
+                        Some('"') => {
+                            self.next().unwrap();
+                            true
+                        }
+                        Some('\'') => {
+                            self.next().unwrap();
+                            match self.peek() {
+                                Some('\'') => {
+                                    self.next().unwrap();
+                                    true
+                                }
+                                _ => false,
+                            }
+                        }
+                        _ => false,
+                    };
+                    return Some(if status { TOKEN_STRING_END } else { TOKEN_ERROR });
+                }
+                _ => (),
+            }
+            break;
+        }
+
+        if self.consume(char::is_whitespace) > 0 {
+            return Some(TOKEN_WHITESPACE);
+        }
+
+        if self.peek() == Some('#') {
+            self.consume(|c| c != '\n');
+            return Some(TOKEN_COMMENT);
+        }
+        if self.starts_with_bump("/*") {
+            loop {
+                self.consume(|c| c != '*');
+                self.next(); // consume the '*', if any
+                match self.peek() {
+                    None => return Some(TOKEN_ERROR),
+                    Some('/') => {
+                        self.next().unwrap();
+                        return Some(TOKEN_COMMENT);
+                    }
+                    _ => (),
+                }
+            }
+        }
+
+        if self.starts_with_bump("...") {
+            return Some(TOKEN_ELLIPSIS);
+        }
+
+        // Check if it's a path
+        // First, simple prefix based detection for absolute ("/...") or home relative ("~/...") paths.
+        // For absolute paths, we need to be careful to not interfere with operators
+        let kind = if self.peek() == Some('/') {
+            // For absolute paths, we need to have a valid path character after the '/'
+            // AND we need to make sure it's not an operator like //
+            if self.remaining().starts_with("//") {
+                None // This could be an update operator, let the operator matcher handle it
+            } else if self.remaining().starts_with("/${") {
+                // Absolute path with interpolation, e.g. /${foo}/bar
+                Some(IdentType::PathAbs)
+            } else {
+                let second_char = self.remaining().chars().nth(1);
+                if second_char.is_some_and(|c| is_valid_path_char(c) && c != '*') {
+                    Some(IdentType::PathAbs)
+                } else {
+                    None // Not a path, might be division operator or something else
+                }
+            }
+        } else if self.peek() == Some('~') && self.remaining().chars().nth(1) == Some('/') {
+            Some(IdentType::PathHome)
+        } else {
+            // Fallback to heuristic previously used for relative, search and URI paths.
+            let store_path = self.peek() == Some('<');
+            let skipped = self
+                .remaining()
+                .chars()
+                .take_while(|&c| match c {
+                    '<' | '/' => store_path,
+                    _ => is_valid_path_char(c),
+                })
+                .collect::<String>();
+
+            let mut lookahead = self.remaining().chars().skip(skipped.chars().count());
+
+            match (lookahead.next(), lookahead.next()) {
+                // a//b parses as Update(a, b)
+                (Some('/'), Some('/')) => None,
+                (Some('/'), Some('*')) => None,
+                (Some('/'), Some(c)) if !c.is_whitespace() => {
+                    // If the first character of the yet‑to‑be‑consumed token was '/', we treat it as absolute later.
+                    // Otherwise it's a relative path.
+                    Some(IdentType::PathRel)
+                }
+                (Some('>'), _) => Some(IdentType::PathSearch),
+                (Some(':'), Some(c)) if is_valid_uri_char(c) && !skipped.contains('_') => {
+                    Some(IdentType::Uri)
+                }
+                _ => None,
+            }
+        };
+
+        let c = self.next()?;
+
+        // First, handle specific operators that might be confused with paths
+        if c == '/' && self.peek() == Some('/') {
+            // This is the update operator '//'
+            self.next().unwrap(); // Consume the second '/'
+            return Some(TOKEN_UPDATE);
+        }
+
+        if c == '~'
+            || matches!(kind, Some(IdentType::PathAbs | IdentType::PathRel))
+            || (c == '/' && self.peek().is_some_and(is_valid_path_char))
+        {
+            return Some({
+                // Determine the concrete path token kind to use
+                let token_kind = if c == '~' {
+                    // we've just consumed '~', ensure '/' exists already consumed by next()
+                    if self.next() != Some('/') {
+                        return Some(TOKEN_ERROR);
+                    }
+                    TOKEN_PATH_HOME
+                } else {
+                    // For abs vs rel: If the first char we consumed was '/', it's absolute.
+                    if c == '/' {
+                        TOKEN_PATH_ABS
+                    } else {
+                        TOKEN_PATH_REL
+                    }
+                };
+
+                // Store the kind in context so subsequent segments are consistent
+                self.push_ctx(Context::Path(token_kind));
+                self.check_path_since(start, token_kind)
+            });
+        }
+
+        Some(match c {
+            '=' if self.peek() == Some('=') => {
+                self.next().unwrap();
+                TOKEN_EQUAL
+            }
+            '!' if self.peek() == Some('=') => {
+                self.next().unwrap();
+                TOKEN_NOT_EQUAL
+            }
+            '!' => TOKEN_INVERT,
+            '{' => {
+                if let Some(Context::Interpol { brackets }) = self.ctx.last_mut() {
+                    *brackets += 1;
+                }
+                TOKEN_L_BRACE
+            }
+            '}' => {
+                if let Some(Context::Interpol { brackets }) = self.ctx.last_mut() {
+                    match brackets.checked_sub(1) {
+                        Some(new) => *brackets = new,
+                        None => {
+                            self.pop_ctx(Context::Interpol { brackets: 0 });
+                            return Some(TOKEN_INTERPOL_END);
+                        }
+                    }
+                }
+                TOKEN_R_BRACE
+            }
+            '[' => TOKEN_L_BRACK,
+            ']' => TOKEN_R_BRACK,
+            '@' => TOKEN_AT,
+            ':' => TOKEN_COLON,
+            ',' => TOKEN_COMMA,
+            '.' => {
+                if self.peek().is_some_and(|x| x.is_ascii_digit()) {
+                    self.consume(|c| c.is_ascii_digit());
+                    self.consume_scientific()
+                } else {
+                    TOKEN_DOT
+                }
+            }
+            '=' => TOKEN_ASSIGN,
+            '?' => TOKEN_QUESTION,
+            ';' => TOKEN_SEMICOLON,
+            '(' => TOKEN_L_PAREN,
+            ')' => TOKEN_R_PAREN,
+            '+' if self.peek() == Some('+') => {
+                self.next().unwrap();
+                TOKEN_CONCAT
+            }
+            '-' if self.peek() == Some('>') => {
+                self.next().unwrap();
+                TOKEN_IMPLICATION
+            }
+            '/' if self.peek() == Some('/') => {
+                self.next().unwrap();
+                TOKEN_UPDATE
+            }
+            '+' => TOKEN_ADD,
+            '-' => TOKEN_SUB,
+            '*' => TOKEN_MUL,
+            '/' => TOKEN_DIV,
+            '<' if self.peek() == Some('|') => {
+                self.next().unwrap();
+                TOKEN_PIPE_LEFT
+            }
+            '<' if kind == Some(IdentType::PathSearch) => {
+                let content_start = self.state.offset;
+                self.consume(is_valid_path_char);
+                // Check if we consumed any content between < and >
+                if self.state.offset == content_start {
+                    // Empty search path <> is not valid
+                    TOKEN_LESS
+                } else if self.next() != Some('>') {
+                    TOKEN_ERROR
+                } else {
+                    TOKEN_PATH_SEARCH
+                }
+            }
+            '&' if self.peek() == Some('&') => {
+                self.next().unwrap();
+                TOKEN_AND_AND
+            }
+            '|' if self.peek() == Some('>') => {
+                self.next().unwrap();
+                TOKEN_PIPE_RIGHT
+            }
+            '|' if self.peek() == Some('|') => {
+                self.next().unwrap();
+                TOKEN_OR_OR
+            }
+            '<' if self.peek() == Some('=') => {
+                self.next().unwrap();
+                TOKEN_LESS_OR_EQ
+            }
+            '<' => TOKEN_LESS,
+            '>' if self.peek() == Some('=') => {
+                self.next().unwrap();
+                TOKEN_MORE_OR_EQ
+            }
+            '>' => TOKEN_MORE,
+            '$' if self.peek() == Some('{') => {
+                self.next().unwrap();
+                self.push_ctx(Context::Interpol { brackets: 0 });
+                TOKEN_INTERPOL_START
+            }
+            'a'..='z' | 'A'..='Z' | '_' => {
+                let kind = match kind {
+                    // It's detected as store if it ends with >, but if it
+                    // didn't start with <, that's wrong
+                    Some(IdentType::PathSearch) | None => IdentType::Ident,
+                    Some(kind) => kind,
+                };
+                self.consume(|c| match c {
+                    'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' | '\'' => true,
+                    c => kind == IdentType::Uri && is_valid_uri_char(c),
+                });
+                match kind {
+                    IdentType::Ident => match self.str_since(start) {
+                        "assert" => TOKEN_ASSERT,
+                        "else" => TOKEN_ELSE,
+                        "if" => TOKEN_IF,
+                        "in" => TOKEN_IN,
+                        "inherit" => TOKEN_INHERIT,
+                        "let" => TOKEN_LET,
+                        // "or" is a contextual keyword and will be handled in the parser.
+                        "or" => TOKEN_OR,
+                        "rec" => TOKEN_REC,
+                        "then" => TOKEN_THEN,
+                        "with" => TOKEN_WITH,
+                        _ => TOKEN_IDENT,
+                    },
+                    IdentType::Uri => TOKEN_URI,
+                    IdentType::PathAbs | IdentType::PathRel | IdentType::PathHome => {
+                        panic!("paths are checked earlier")
+                    }
+                    IdentType::PathSearch => panic!("search paths are checked earlier"),
+                }
+            }
+            '"' => {
+                self.push_ctx(Context::StringBody { multiline: false });
+                TOKEN_STRING_START
+            }
+            '\'' if self.peek() == Some('\'') => {
+                self.next().unwrap();
+                self.push_ctx(Context::StringBody { multiline: true });
+                TOKEN_STRING_START
+            }
+            fst @ '0'..='9' => {
+                // Due to nix's behavior on leading 0's we know immediately
+                // That a '0' followed by some other number character MUST tokenize into an INT.
+                // More background on this: https://github.com/nix-community/rnix-parser/issues/157
+                if fst == '0' && self.peek() != Some('.') {
+                    // In that case just consume the remaining number characters to fill the TOKEN_INTEGER
+                    self.consume(|c| c.is_ascii_digit());
+                    TOKEN_INTEGER
+                } else {
+                    // If we are not in that edge-case we can just continue with the regular number tokenization
+                    self.consume(|c| c.is_ascii_digit());
+                    if self.peek() == Some('.') {
+                        self.next().unwrap();
+                        self.consume(|c| c.is_ascii_digit());
+                        self.consume_scientific()
+                    } else {
+                        TOKEN_INTEGER
+                    }
+                }
+            }
+            _ => TOKEN_ERROR,
+        })
+    }
+
+    fn consume_scientific(&mut self) -> SyntaxKind {
+        if self.peek() == Some('e') || self.peek() == Some('E') {
+            self.next().unwrap();
+            if self.peek() == Some('-') || self.peek() == Some('+') {
+                self.next().unwrap();
+            }
+            if self.consume(|c| c.is_ascii_digit()) == 0 {
+                return TOKEN_ERROR;
+            }
+        }
+        TOKEN_FLOAT
+    }
+}
+
+impl<'a> Iterator for Tokenizer<'a> {
+    type Item = Token<'a>;
+    fn next(&mut self) -> Option<Self::Item> {
+        let start = self.state;
+        self.next_inner().map(|syntax_kind| (syntax_kind, self.str_since(start)))
+    }
+}