feat: support multiple text nodes

Author: Magda98

src/config.ts

@@ -4,6 +4,5 @@ import type { Config } from './syntax-highlight-component.types';
 export const configDefaults: Config = {
   languages: ['markup', 'css', 'html', 'javascript', 'typescript'],
   tokenTypes,
-  languageTokens: {},
   tokenize,
 };

src/cssHighlight.ts

@@ -1,32 +1,13 @@
 /**
  * Initializes CSS Custom Highlights for a given set of token types.
  *
- * This function iterates through a list of common token types and optional
- * language-specific token types, creating and registering a new `Highlight`
- * object for each one in the `CSS.highlights` registry. This pre-registration
- * is a necessary step for applying styles to code tokens using the
- * CSS Custom Highlight API.
- *
  * @param tokenTypes An array of strings representing common token types (e.g., 'keyword', 'string').
- * @param options An object containing additional configuration.
- * @param options.languageTokens An optional object mapping language identifiers to their specific token types.
  * These are registered with a name in the format `${language}-${tokenType}`.
  * ```
  * @see https://developer.mozilla.org/en-US/docs/Web/API/CSS_Custom_Highlight_API
  */
-export function setupTokenHighlights(
-  tokenTypes: string[],
-  { languageTokens }: { languageTokens?: { [language: string]: string[] } },
-) {
+export function setupTokenHighlights(tokenTypes: string[]) {
   for (const tokenType of tokenTypes) {
     CSS.highlights.set(tokenType, new Highlight());
   }
-
-  if (languageTokens) {
-    for (const language in languageTokens) {
-      for (const tokenType of languageTokens[language]) {
-        CSS.highlights.set(`${language}-${tokenType}`, new Highlight());
-      }
-    }
-  }
 }

src/syntax-highlight-component.ts

@@ -5,8 +5,8 @@ import {
   type Language,
   type Theme,
 } from './syntax-highlight-component.types';
-import { setupTokenHighlights } from './cssHighlight';
 import { configDefaults } from './config';
+import { setupTokenHighlights } from './cssHighlight';
 
 /**
  * A web component for syntax highlighting code blocks using the CSS Custom Highlight API.
@@ -80,9 +80,7 @@ export default class SyntaxHighlightComponent extends LitElement {
     super();
     this._internals = this.attachInternals();
     this._internals.role = 'code';
-    setupTokenHighlights(configDefaults.tokenTypes, {
-      languageTokens: configDefaults.languageTokens,
-    });
+    setupTokenHighlights(configDefaults.tokenTypes);
   }
 
   get contentElement(): HTMLElement {
@@ -223,36 +221,30 @@ export default class SyntaxHighlightComponent extends LitElement {
     return html`<slot @slotchange=${this.paintTokenHighlights}></slot>`;
   }
 
-  async paintTokenHighlights() {
+  paintTokenHighlights() {
     this.clearTokenHighlights();
     if (!CSS.highlights) return;
 
-    const text = this.contentElement.innerText;
-    const tokens = await SyntaxHighlightComponent._config.tokenize(
-      text,
-      this.language,
-    );
-    const languageTokenTypes =
-      SyntaxHighlightComponent._config.languageTokens?.[this.language] || [];
-
-    let pos = 0;
-    for (const token of tokens) {
-      if (token.type) {
-        const tokenType = languageTokenTypes.includes(token.type)
-          ? `${this.language}-${token.type}`
-          : token.type;
-
-        const range = new Range();
-        if (this.contentElement.firstChild) {
-          range.setStart(this.contentElement.firstChild, pos);
-          range.setEnd(this.contentElement.firstChild, pos + token.length);
+    this.contentElement.childNodes.forEach(async (node) => {
+      let pos = 0;
+      const tokens = await SyntaxHighlightComponent._config.tokenize(
+        node.textContent || '',
+        this.language,
+      );
+      for (const token of tokens) {
+        if (token.type) {
+          const range = new Range();
+          if (node && node.textContent?.length) {
+            range.setStart(node, pos);
+            range.setEnd(node, pos + token.length);
 
-          CSS.highlights.get(tokenType)?.add(range);
-          this._highlights.add({ tokenType, range });
+            CSS.highlights.get(token.type)?.add(range);
+            this._highlights.add({ tokenType: token.type, range });
+          }
         }
+        pos += token.length;
       }
-      pos += token.length;
-    }
+    });
   }
 
   protected clearTokenHighlights() {

src/syntax-highlight-component.types.ts

@@ -1,7 +1,6 @@
 export type Config = {
   languages: Language[];
   tokenTypes: string[];
-  languageTokens: Record<string, string[]>;
   tokenize: (text: string, language: Language) => Promise<FlatToken[]>;
 };