nifty_lsp/features/
semantic_tokens.rs

1//! Semantic tokens for names, which the `TextMate` grammar can't tell apart: a struct from a
2//! view, a state value from a local, a built-in element from a declared function. Keywords,
3//! literals, operators and comments are left to the grammar.
4
5use lsp_types::{SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend};
6
7use nifty_check::hir::Type;
8use nifty_check::{elements, functions, methods};
9
10use nifty_syntax::FileId;
11
12use crate::document::Encoding;
13use crate::index::{Index, Role, SymbolKind, Unresolved};
14use crate::project::Project;
15
16#[derive(Clone, Copy)]
17enum TokenType {
18    Struct,
19    View,
20    Function,
21    Method,
22    Variable,
23    Parameter,
24    Property,
25    EnumMember,
26    Type,
27}
28
29impl TokenType {
30    const ALL: [Self; 9] = [
31        Self::Struct,
32        Self::View,
33        Self::Function,
34        Self::Method,
35        Self::Variable,
36        Self::Parameter,
37        Self::Property,
38        Self::EnumMember,
39        Self::Type,
40    ];
41
42    fn lsp(self) -> SemanticTokenType {
43        match self {
44            Self::Struct => SemanticTokenType::STRUCT,
45            // Views are used like components, which editors theme as classes.
46            Self::View => SemanticTokenType::CLASS,
47            Self::Function => SemanticTokenType::FUNCTION,
48            Self::Method => SemanticTokenType::METHOD,
49            Self::Variable => SemanticTokenType::VARIABLE,
50            Self::Parameter => SemanticTokenType::PARAMETER,
51            Self::Property => SemanticTokenType::PROPERTY,
52            Self::EnumMember => SemanticTokenType::ENUM_MEMBER,
53            Self::Type => SemanticTokenType::TYPE,
54        }
55    }
56}
57
58#[derive(Clone, Copy)]
59enum Modifier {
60    Declaration,
61    Readonly,
62    DefaultLibrary,
63}
64
65impl Modifier {
66    const ALL: [Self; 3] = [Self::Declaration, Self::Readonly, Self::DefaultLibrary];
67
68    fn lsp(self) -> SemanticTokenModifier {
69        match self {
70            Self::Declaration => SemanticTokenModifier::DECLARATION,
71            Self::Readonly => SemanticTokenModifier::READONLY,
72            Self::DefaultLibrary => SemanticTokenModifier::DEFAULT_LIBRARY,
73        }
74    }
75
76    fn bit(self) -> u32 {
77        1 << self as u32
78    }
79}
80
81pub(crate) fn legend() -> SemanticTokensLegend {
82    SemanticTokensLegend {
83        token_types: TokenType::ALL.map(TokenType::lsp).to_vec(),
84        token_modifiers: Modifier::ALL.map(Modifier::lsp).to_vec(),
85    }
86}
87
88/// Every name in `file`, encoded relative to the one before it as the protocol asks.
89pub(crate) fn semantic_tokens(
90    project: &Project,
91    file: FileId,
92    encoding: Encoding,
93) -> Vec<SemanticToken> {
94    let document = project.document(file);
95    let mut previous_line = 0;
96    let mut previous_start = 0;
97    project
98        .index
99        .occurrences_in(file)
100        .iter()
101        .map(|occurrence| {
102            let start = document.position(occurrence.span.start, encoding);
103            let delta_line = start.line - previous_line;
104            let delta_start = if delta_line == 0 {
105                start.character - previous_start
106            } else {
107                start.character
108            };
109            previous_line = start.line;
110            previous_start = start.character;
111
112            let (token_type, modifiers) = classify(
113                &project.index,
114                occurrence.role,
115                document.text(occurrence.span),
116            );
117            SemanticToken {
118                delta_line,
119                delta_start,
120                length: encoding.len(document.text(occurrence.span)),
121                token_type: token_type as u32,
122                token_modifiers_bitset: modifiers,
123            }
124        })
125        .collect()
126}
127
128fn classify(index: &Index, role: Role, name: &str) -> (TokenType, u32) {
129    match role {
130        Role::Declaration(id) => {
131            let (token_type, modifiers) = classify_symbol(index.symbol(id).kind);
132            (token_type, modifiers | Modifier::Declaration.bit())
133        }
134        Role::Reference(id) => classify_symbol(index.symbol(id).kind),
135        Role::Unresolved(unresolved) => match unresolved {
136            Unresolved::Function => (
137                TokenType::Function,
138                built_in(elements::element(name).is_some() || functions::function(name).is_some()),
139            ),
140            Unresolved::Type => (TokenType::Type, built_in(Type::primitive(name).is_some())),
141            Unresolved::Variable => (TokenType::Variable, 0),
142            Unresolved::Field => (TokenType::Property, 0),
143            Unresolved::Method => (TokenType::Method, built_in(methods::method(name).is_some())),
144            Unresolved::ImplicitMember => (TokenType::EnumMember, 0),
145        },
146    }
147}
148
149/// The modifier for a name the file doesn't declare: from the built-in tables, or none
150/// for a misspelling.
151fn built_in(is_built_in: bool) -> u32 {
152    if is_built_in {
153        Modifier::DefaultLibrary.bit()
154    } else {
155        0
156    }
157}
158
159/// A symbol's token type, marked readonly when the name can't be assigned.
160fn classify_symbol(kind: SymbolKind) -> (TokenType, u32) {
161    let readonly = Modifier::Readonly.bit();
162    match kind {
163        SymbolKind::State => (TokenType::Variable, 0),
164        SymbolKind::Local | SymbolKind::Window => (TokenType::Variable, readonly),
165        SymbolKind::Parameter => (TokenType::Parameter, readonly),
166        SymbolKind::Struct => (TokenType::Struct, 0),
167        SymbolKind::Field => (TokenType::Property, 0),
168        SymbolKind::Function => (TokenType::Function, 0),
169        SymbolKind::View => (TokenType::View, 0),
170    }
171}
172
173#[cfg(test)]
174mod tests {
175    use super::*;
176
177    #[test]
178    fn encodes_names_relative_to_the_previous_one() {
179        let project = Project::single("state n = 0\nlet m = n");
180        let tokens: Vec<_> = semantic_tokens(&project, FileId::default(), Encoding::Utf16)
181            .into_iter()
182            .map(|t| {
183                (
184                    t.delta_line,
185                    t.delta_start,
186                    t.length,
187                    t.token_type,
188                    t.token_modifiers_bitset,
189                )
190            })
191            .collect();
192        let variable = TokenType::Variable as u32;
193        let declaration = Modifier::Declaration.bit();
194        let readonly = Modifier::Readonly.bit();
195        assert_eq!(
196            tokens,
197            [
198                (0, 6, 1, variable, declaration),
199                (1, 4, 1, variable, declaration | readonly),
200                (0, 4, 1, variable, 0),
201            ]
202        );
203    }
204
205    #[test]
206    fn only_names_in_the_built_in_tables_are_marked_default_library() {
207        let source = "fn f(xs: [int], t: strng) {\n  xs.push(1)\n  xs.pushh(1)\n  download(\"u\")\n  downlod(\"u\")\n}";
208        let project = Project::single(source);
209        let marked: Vec<_> = project
210            .index
211            .occurrences_in(FileId::default())
212            .iter()
213            .filter_map(|occurrence| {
214                let name = project.text(occurrence.span);
215                let (_, modifiers) = classify(&project.index, occurrence.role, name);
216                (modifiers & Modifier::DefaultLibrary.bit() != 0).then_some(name)
217            })
218            .collect();
219        assert_eq!(marked, ["int", "push", "download"]);
220    }
221}
nifty_lsp/features/semantic_tokens.rs

nifty_lsp/features/
semantic_tokens.rs
