Auto merge of #13084 - Veykril:highlight-config, r=Veykril

Add some more highlighting configurations

The following can be enabled/disabled now in terms of highlighting:
- doc comment injection (enabled by default)
- punctuation highlighting (disabled by default)
- operator highlighting (enabled by default)
- punctuation specialized highlighting (disabled by default)
- operator specialized highlighting (disabled by default)
- macro call bang highlighting (disabled by default)

This PR also changes our `attribute` semantic token type to the `decorator` type which landed upstream (but not yet in lsp-types).

Specialized highlighting is disabled by default, as all clients will have to ship something to map these back to the standard punctuation/operator token (we do this in VSCode via the inheritance mapping for example). This is a lot of maintenance work, and not something every client wants to do, pushing that need to use the user. As this is a rather niche use in the first place this will just be disabled by default.

Punctuation highlighting is disabled by default, punctuation is usually something that can be done by the native syntactic highlighting of the client, so there is no loss in quality. The main reason for this though is that punctuation adds a lot of extra token data that we sent over, a lot of clients struggle with applying this, so disabling this improves the UX for a lot of people. Note that we still highlight punctuations with special meaning as that special entity, (the never type `!` will still be tagged as a builtin type if it occurs as such)

Separate highlighting of the macro call bang `!` is disabled by default, as I think people actually didn't like that change that much, though at the same time I feel like not many people even noticed that change (I prefer it be separate, but that's not enough reason for it to be enabled by default I believe :^) )

cc #12783 #13066

Author: bors

Cargo.lock

@@ -98,7 +98,7 @@ pub use crate::{
     static_index::{StaticIndex, StaticIndexedFile, TokenId, TokenStaticData},
     syntax_highlighting::{
         tags::{Highlight, HlMod, HlMods, HlOperator, HlPunct, HlTag},
-        HlRange,
+        HighlightConfig, HlRange,
     },
 };
 pub use hir::{Documentation, Semantics};
@@ -517,8 +517,12 @@ impl Analysis {
     }
 
     /// Computes syntax highlighting for the given file
-    pub fn highlight(&self, file_id: FileId) -> Cancellable<Vec<HlRange>> {
-        self.with_db(|db| syntax_highlighting::highlight(db, file_id, None, false))
+    pub fn highlight(
+        &self,
+        highlight_config: HighlightConfig,
+        file_id: FileId,
+    ) -> Cancellable<Vec<HlRange>> {
+        self.with_db(|db| syntax_highlighting::highlight(db, highlight_config, file_id, None))
     }
 
     /// Computes all ranges to highlight for a given item in a file.
@@ -533,9 +537,13 @@ impl Analysis {
     }
 
     /// Computes syntax highlighting for the given file range.
-    pub fn highlight_range(&self, frange: FileRange) -> Cancellable<Vec<HlRange>> {
+    pub fn highlight_range(
+        &self,
+        highlight_config: HighlightConfig,
+        frange: FileRange,
+    ) -> Cancellable<Vec<HlRange>> {
         self.with_db(|db| {
-            syntax_highlighting::highlight(db, frange.file_id, Some(frange.range), false)
+            syntax_highlighting::highlight(db, highlight_config, frange.file_id, Some(frange.range))
         })
     }
 

crates/ide/src/lib.rs

@@ -14,7 +14,7 @@ mod html;
 mod tests;
 
 use hir::{Name, Semantics};
-use ide_db::{FxHashMap, RootDatabase};
+use ide_db::{FxHashMap, RootDatabase, SymbolKind};
 use syntax::{
     ast, AstNode, AstToken, NodeOrToken, SyntaxKind::*, SyntaxNode, TextRange, WalkEvent, T,
 };
@@ -24,7 +24,7 @@ use crate::{
         escape::highlight_escape_string, format::highlight_format_string, highlights::Highlights,
         macro_::MacroHighlighter, tags::Highlight,
     },
-    FileId, HlMod, HlTag,
+    FileId, HlMod, HlOperator, HlPunct, HlTag,
 };
 
 pub(crate) use html::highlight_as_html;
@@ -36,6 +36,26 @@ pub struct HlRange {
     pub binding_hash: Option<u64>,
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct HighlightConfig {
+    /// Whether to highlight strings
+    pub strings: bool,
+    /// Whether to highlight punctuation
+    pub punctuation: bool,
+    /// Whether to specialize punctuation highlights
+    pub specialize_punctuation: bool,
+    /// Whether to highlight operator
+    pub operator: bool,
+    /// Whether to specialize operator highlights
+    pub specialize_operator: bool,
+    /// Whether to inject highlights into doc comments
+    pub inject_doc_comment: bool,
+    /// Whether to highlight the macro call bang
+    pub macro_bang: bool,
+    /// Whether to highlight unresolved things be their syntax
+    pub syntactic_name_ref_highlighting: bool,
+}
+
 // Feature: Semantic Syntax Highlighting
 //
 // rust-analyzer highlights the code semantically.
@@ -155,9 +175,9 @@ pub struct HlRange {
 // image::https://user-images.githubusercontent.com/48062697/113187625-f7f50100-9250-11eb-825e-91c58f236071.png[]
 pub(crate) fn highlight(
     db: &RootDatabase,
+    config: HighlightConfig,
     file_id: FileId,
     range_to_highlight: Option<TextRange>,
-    syntactic_name_ref_highlighting: bool,
 ) -> Vec<HlRange> {
     let _p = profile::span("highlight");
     let sema = Semantics::new(db);
@@ -183,26 +203,18 @@ pub(crate) fn highlight(
         Some(it) => it.krate(),
         None => return hl.to_vec(),
     };
-    traverse(
-        &mut hl,
-        &sema,
-        file_id,
-        &root,
-        krate,
-        range_to_highlight,
-        syntactic_name_ref_highlighting,
-    );
+    traverse(&mut hl, &sema, config, file_id, &root, krate, range_to_highlight);
     hl.to_vec()
 }
 
 fn traverse(
     hl: &mut Highlights,
     sema: &Semantics<'_, RootDatabase>,
+    config: HighlightConfig,
     file_id: FileId,
     root: &SyntaxNode,
     krate: hir::Crate,
     range_to_highlight: TextRange,
-    syntactic_name_ref_highlighting: bool,
 ) {
     let is_unlinked = sema.to_module_def(file_id).is_none();
     let mut bindings_shadow_count: FxHashMap<Name, u32> = FxHashMap::default();
@@ -323,9 +335,11 @@ fn traverse(
             Enter(it) => it,
             Leave(NodeOrToken::Token(_)) => continue,
             Leave(NodeOrToken::Node(node)) => {
-                // Doc comment highlighting injection, we do this when leaving the node
-                // so that we overwrite the highlighting of the doc comment itself.
-                inject::doc_comment(hl, sema, file_id, &node);
+                if config.inject_doc_comment {
+                    // Doc comment highlighting injection, we do this when leaving the node
+                    // so that we overwrite the highlighting of the doc comment itself.
+                    inject::doc_comment(hl, sema, config, file_id, &node);
+                }
                 continue;
             }
         };
@@ -400,7 +414,8 @@ fn traverse(
                 let string_to_highlight = ast::String::cast(descended_token.clone());
                 if let Some((string, expanded_string)) = string.zip(string_to_highlight) {
                     if string.is_raw() {
-                        if inject::ra_fixture(hl, sema, &string, &expanded_string).is_some() {
+                        if inject::ra_fixture(hl, sema, config, &string, &expanded_string).is_some()
+                        {
                             continue;
                         }
                     }
@@ -421,7 +436,7 @@ fn traverse(
                 sema,
                 krate,
                 &mut bindings_shadow_count,
-                syntactic_name_ref_highlighting,
+                config.syntactic_name_ref_highlighting,
                 name_like,
             ),
             NodeOrToken::Token(token) => highlight::token(sema, token).zip(Some(None)),
@@ -439,6 +454,29 @@ fn traverse(
                 // something unresolvable. FIXME: There should be a way to prevent that
                 continue;
             }
+
+            // apply config filtering
+            match &mut highlight.tag {
+                HlTag::StringLiteral if !config.strings => continue,
+                // If punctuation is disabled, make the macro bang part of the macro call again.
+                tag @ HlTag::Punctuation(HlPunct::MacroBang) => {
+                    if !config.macro_bang {
+                        *tag = HlTag::Symbol(SymbolKind::Macro);
+                    } else if !config.specialize_punctuation {
+                        *tag = HlTag::Punctuation(HlPunct::Other);
+                    }
+                }
+                HlTag::Punctuation(_) if !config.punctuation => continue,
+                tag @ HlTag::Punctuation(_) if !config.specialize_punctuation => {
+                    *tag = HlTag::Punctuation(HlPunct::Other);
+                }
+                HlTag::Operator(_) if !config.operator && highlight.mods.is_empty() => continue,
+                tag @ HlTag::Operator(_) if !config.specialize_operator => {
+                    *tag = HlTag::Operator(HlOperator::Other);
+                }
+                _ => (),
+            }
+
             if inside_attribute {
                 highlight |= HlMod::Attribute
             }

crates/ide/src/syntax_highlighting.rs

@@ -5,7 +5,10 @@ use oorandom::Rand32;
 use stdx::format_to;
 use syntax::AstNode;
 
-use crate::{syntax_highlighting::highlight, FileId, RootDatabase};
+use crate::{
+    syntax_highlighting::{highlight, HighlightConfig},
+    FileId, RootDatabase,
+};
 
 pub(crate) fn highlight_as_html(db: &RootDatabase, file_id: FileId, rainbow: bool) -> String {
     let parse = db.parse(file_id);
@@ -20,7 +23,21 @@ pub(crate) fn highlight_as_html(db: &RootDatabase, file_id: FileId, rainbow: boo
         )
     }
 
-    let hl_ranges = highlight(db, file_id, None, false);
+    let hl_ranges = highlight(
+        db,
+        HighlightConfig {
+            strings: true,
+            punctuation: true,
+            specialize_punctuation: true,
+            specialize_operator: true,
+            operator: true,
+            inject_doc_comment: true,
+            macro_bang: true,
+            syntactic_name_ref_highlighting: false,
+        },
+        file_id,
+        None,
+    );
     let text = parse.tree().syntax().to_string();
     let mut buf = String::new();
     buf.push_str(STYLE);

crates/ide/src/syntax_highlighting/html.rs

@@ -15,13 +15,14 @@ use syntax::{
 
 use crate::{
     doc_links::{doc_attributes, extract_definitions_from_docs, resolve_doc_path_for_def},
-    syntax_highlighting::{highlights::Highlights, injector::Injector},
+    syntax_highlighting::{highlights::Highlights, injector::Injector, HighlightConfig},
     Analysis, HlMod, HlRange, HlTag, RootDatabase,
 };
 
 pub(super) fn ra_fixture(
     hl: &mut Highlights,
     sema: &Semantics<'_, RootDatabase>,
+    config: HighlightConfig,
     literal: &ast::String,
     expanded: &ast::String,
 ) -> Option<()> {
@@ -63,7 +64,13 @@ pub(super) fn ra_fixture(
 
     let (analysis, tmp_file_id) = Analysis::from_single_file(inj.take_text());
 
-    for mut hl_range in analysis.highlight(tmp_file_id).unwrap() {
+    for mut hl_range in analysis
+        .highlight(
+            HighlightConfig { syntactic_name_ref_highlighting: false, ..config },
+            tmp_file_id,
+        )
+        .unwrap()
+    {
         for range in inj.map_range_up(hl_range.range) {
             if let Some(range) = literal.map_range_up(range) {
                 hl_range.range = range;
@@ -86,6 +93,7 @@ const RUSTDOC_FENCES: [&str; 2] = ["```", "~~~"];
 pub(super) fn doc_comment(
     hl: &mut Highlights,
     sema: &Semantics<'_, RootDatabase>,
+    config: HighlightConfig,
     src_file_id: FileId,
     node: &SyntaxNode,
 ) {
@@ -206,7 +214,14 @@ pub(super) fn doc_comment(
 
     let (analysis, tmp_file_id) = Analysis::from_single_file(inj.take_text());
 
-    if let Ok(ranges) = analysis.with_db(|db| super::highlight(db, tmp_file_id, None, true)) {
+    if let Ok(ranges) = analysis.with_db(|db| {
+        super::highlight(
+            db,
+            HighlightConfig { syntactic_name_ref_highlighting: true, ..config },
+            tmp_file_id,
+            None,
+        )
+    }) {
         for HlRange { range, highlight, binding_hash } in ranges {
             for range in inj.map_range_up(range) {
                 hl.add(HlRange { range, highlight: highlight | HlMod::Injected, binding_hash });

crates/ide/src/syntax_highlighting/inject.rs

@@ -199,7 +199,7 @@ impl fmt::Display for HlTag {
 }
 
 impl HlMod {
-    const ALL: &'static [HlMod; HlMod::Unsafe as u8 as usize + 1] = &[
+    const ALL: &'static [HlMod; 19] = &[
         HlMod::Associated,
         HlMod::Async,
         HlMod::Attribute,
@@ -296,7 +296,7 @@ impl Highlight {
         Highlight { tag, mods: HlMods::default() }
     }
     pub fn is_empty(&self) -> bool {
-        self.tag == HlTag::None && self.mods == HlMods::default()
+        self.tag == HlTag::None && self.mods.is_empty()
     }
 }
 
@@ -330,6 +330,10 @@ impl ops::BitOr<HlMod> for Highlight {
 }
 
 impl HlMods {
+    pub fn is_empty(&self) -> bool {
+        self.0 == 0
+    }
+
     pub fn contains(self, m: HlMod) -> bool {
         self.0 & m.mask() == m.mask()
     }

crates/ide/src/syntax_highlighting/tags.rs

@@ -4,7 +4,18 @@ use expect_test::{expect_file, ExpectFile};
 use ide_db::SymbolKind;
 use test_utils::{bench, bench_fixture, skip_slow_tests, AssertLinear};
 
-use crate::{fixture, FileRange, HlTag, TextRange};
+use crate::{fixture, FileRange, HighlightConfig, HlTag, TextRange};
+
+const HL_CONFIG: HighlightConfig = HighlightConfig {
+    strings: true,
+    punctuation: true,
+    specialize_punctuation: true,
+    specialize_operator: true,
+    operator: true,
+    inject_doc_comment: true,
+    macro_bang: true,
+    syntactic_name_ref_highlighting: false,
+};
 
 #[test]
 fn attributes() {
@@ -996,7 +1007,10 @@ struct Foo {
 
     // The "x"
     let highlights = &analysis
-        .highlight_range(FileRange { file_id, range: TextRange::at(45.into(), 1.into()) })
+        .highlight_range(
+            HL_CONFIG,
+            FileRange { file_id, range: TextRange::at(45.into(), 1.into()) },
+        )
         .unwrap();
 
     assert_eq!(&highlights[0].highlight.to_string(), "field.declaration.public");
@@ -1011,7 +1025,7 @@ macro_rules! test {}
 }"#
         .trim(),
     );
-    let _ = analysis.highlight(file_id).unwrap();
+    let _ = analysis.highlight(HL_CONFIG, file_id).unwrap();
 }
 
 /// Highlights the code given by the `ra_fixture` argument, renders the
@@ -1035,7 +1049,7 @@ fn benchmark_syntax_highlighting_long_struct() {
     let hash = {
         let _pt = bench("syntax highlighting long struct");
         analysis
-            .highlight(file_id)
+            .highlight(HL_CONFIG, file_id)
             .unwrap()
             .iter()
             .filter(|it| it.highlight.tag == HlTag::Symbol(SymbolKind::Struct))
@@ -1061,7 +1075,7 @@ fn syntax_highlighting_not_quadratic() {
             let time = Instant::now();
 
             let hash = analysis
-                .highlight(file_id)
+                .highlight(HL_CONFIG, file_id)
                 .unwrap()
                 .iter()
                 .filter(|it| it.highlight.tag == HlTag::Symbol(SymbolKind::Struct))
@@ -1086,7 +1100,7 @@ fn benchmark_syntax_highlighting_parser() {
     let hash = {
         let _pt = bench("syntax highlighting parser");
         analysis
-            .highlight(file_id)
+            .highlight(HL_CONFIG, file_id)
             .unwrap()
             .iter()
             .filter(|it| it.highlight.tag == HlTag::Symbol(SymbolKind::Function))