Add TCL syntax highlighting

llamafile/highlight.cpp

@@ -102,6 +102,8 @@ Highlight *Highlight::create(const std::string_view &lang) {
         return new HighlightC(is_keyword_d);
     } else if (lang == "zig") {
         return new HighlightZig();
+    } else if (lang == "tcl") {
+        return new HighlightTcl();
     } else {
         return new HighlightPlain;
     }

llamafile/highlight.h

@@ -80,6 +80,9 @@ is_keyword_f is_keyword_zig;
 is_keyword_f is_keyword_zig_type;
 is_keyword_f is_keyword_zig_builtin;
 is_keyword_f is_keyword_zig_constant;
+is_keyword_f is_keyword_tcl;
+is_keyword_f is_keyword_tcl_type;
+is_keyword_f is_keyword_tcl_builtin;
 }
 
 class Highlight {
@@ -334,3 +337,15 @@ class HighlightZig : public Highlight {
     int t_ = 0;
     std::string word_;
 };
+
+class HighlightTcl : public Highlight {
+  public:
+    HighlightTcl();
+    ~HighlightTcl() override;
+    void feed(std::string *result, std::string_view input) override;
+    void flush(std::string *result) override;
+
+  private:
+    int t_ = 0;
+    std::string word_;
+};

llamafile/highlight_tcl.cpp

@@ -0,0 +1,173 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+    NORMAL,
+    WORD,
+    DQUOTE,
+    DQUOTE_BACKSLASH,
+    VAR,
+    VAR2,
+    COMMENT,
+};
+
+HighlightTcl::HighlightTcl() {
+}
+
+HighlightTcl::~HighlightTcl() {
+}
+
+void HighlightTcl::feed(std::string *r, std::string_view input) {
+    int c;
+    for (size_t i = 0; i < input.size(); ++i) {
+        c = input[i] & 255;
+        switch (t_) {
+
+        Normal:
+        case NORMAL:
+            if (!isascii(c) || isalpha(c) || c == '_') {
+                t_ = WORD;
+                word_ += c;
+            } else if (c == '"') {
+                t_ = DQUOTE;
+                *r += HI_STRING;
+                *r += c;
+            } else if (c == '$') {
+                *r += c;
+                t_ = VAR;
+            } else if (c == '#') {
+                *r += HI_COMMENT;
+                *r += c;
+                t_ = COMMENT;
+            } else {
+                *r += c;
+            }
+            break;
+
+        case WORD:
+            if (!isascii(c) || isalnum(c) || c == '_') {
+                word_ += c;
+            } else {
+                if (is_keyword_tcl(word_.data(), word_.size())) {
+                    *r += HI_KEYWORD;
+                    *r += word_;
+                    *r += HI_RESET;
+                } else if (is_keyword_tcl_type(word_.data(), word_.size())) {
+                    *r += HI_TYPE;
+                    *r += word_;
+                    *r += HI_RESET;
+                } else if (is_keyword_tcl_builtin(word_.data(), word_.size())) {
+                    *r += HI_BUILTIN;
+                    *r += word_;
+                    *r += HI_RESET;
+                } else {
+                    *r += word_;
+                }
+                word_.clear();
+                t_ = NORMAL;
+                goto Normal;
+            }
+            break;
+
+        case VAR:
+            if (c == '{') {
+                *r += c;
+                *r += HI_VAR;
+                t_ = VAR2;
+                break;
+            } else {
+                *r += HI_VAR;
+                t_ = VAR2;
+            }
+            // fallthrough
+
+        case VAR2:
+            if (!isascii(c) || isalnum(c) || c == '_') {
+                *r += c;
+            } else {
+                *r += HI_RESET;
+                t_ = NORMAL;
+                goto Normal;
+            }
+            break;
+
+        case COMMENT:
+            if (c == '\n') {
+                *r += HI_RESET;
+                *r += c;
+                t_ = NORMAL;
+            } else {
+                *r += c;
+            }
+            break;
+
+        case DQUOTE:
+            *r += c;
+            if (c == '"') {
+                *r += HI_RESET;
+                t_ = NORMAL;
+            } else if (c == '\\') {
+                t_ = DQUOTE_BACKSLASH;
+            }
+            break;
+
+        case DQUOTE_BACKSLASH:
+            *r += c;
+            t_ = DQUOTE;
+            break;
+
+        default:
+            __builtin_unreachable();
+        }
+    }
+}
+
+void HighlightTcl::flush(std::string *r) {
+    switch (t_) {
+    case WORD:
+        if (is_keyword_tcl(word_.data(), word_.size())) {
+            *r += HI_KEYWORD;
+            *r += word_;
+            *r += HI_RESET;
+        } else if (is_keyword_tcl_type(word_.data(), word_.size())) {
+            *r += HI_TYPE;
+            *r += word_;
+            *r += HI_RESET;
+        } else if (is_keyword_tcl_builtin(word_.data(), word_.size())) {
+            *r += HI_BUILTIN;
+            *r += word_;
+            *r += HI_RESET;
+        } else {
+            *r += word_;
+        }
+        word_.clear();
+        break;
+    case VAR2:
+    case DQUOTE:
+    case DQUOTE_BACKSLASH:
+    case COMMENT:
+        *r += HI_RESET;
+        break;
+    default:
+        break;
+    }
+    t_ = NORMAL;
+}

llamafile/highlight_test.cpp

@@ -53,6 +53,7 @@ const char *const kLanguages[] = {
     "rust", //
     "shell", //
     "sql", //
+    "tcl", //
     "swift", //
     "zig", //
 };