feat: add EJS (Embedded JavaScript) templating engine support (#956)

Author: VAIBHAVSING

.changeset/add-ejs-support.md

@@ -0,0 +1,12 @@
+---
+"lingo.dev": minor
+"@lingo.dev/_spec": minor
+---
+
+feat: add EJS (Embedded JavaScript) templating engine support
+
+- Added EJS loader to support parsing and translating EJS template files
+- EJS loader extracts translatable text while preserving EJS tags and expressions
+- Updated spec package to include "ejs" in supported bucket types
+- Added comprehensive test suite covering various EJS scenarios including conditionals, loops, includes, and mixed content
+- Automatically installed EJS dependency (@types/ejs) for TypeScript support

packages/cli/package.json

@@ -130,6 +130,7 @@
     "@modelcontextprotocol/sdk": "^1.5.0",
     "@openrouter/ai-sdk-provider": "^0.7.1",
     "@paralleldrive/cuid2": "^2.2.2",
+    "@types/ejs": "^3.1.5",
     "ai": "^4.3.15",
     "bitbucket": "^2.12.0",
     "chalk": "^5.4.1",
@@ -142,6 +143,7 @@
     "dedent": "^1.5.3",
     "diff": "^7.0.0",
     "dotenv": "^16.4.7",
+    "ejs": "^3.1.10",
     "express": "^5.1.0",
     "external-editor": "^3.1.0",
     "figlet": "^1.8.0",

packages/cli/src/cli/loaders/ejs.spec.ts

@@ -0,0 +1,165 @@
+import { describe, it, expect } from "vitest";
+import createEjsLoader from "./ejs";
+
+describe("EJS Loader", () => {
+  const loader = createEjsLoader().setDefaultLocale("en");
+
+  describe("pull", () => {
+    it("should extract translatable text from simple EJS template", async () => {
+      const input = `
+        <h1>Welcome to our website</h1>
+        <p>Hello <%= name %>, you have <%= messages.length %> messages.</p>
+        <footer>© 2024 Our Company</footer>
+      `;
+
+      const result = await loader.pull("en", input);
+      
+      // Check that we have extracted some translatable content
+      expect(Object.keys(result).length).toBeGreaterThan(0);
+      
+      // Check that the EJS variables are not included in the translatable text
+      const allValues = Object.values(result).join(' ');
+      expect(allValues).not.toContain('<%= name %>');
+      expect(allValues).not.toContain('<%= messages.length %>');
+      
+      // Check that we have the main content
+      expect(allValues).toContain('Welcome to our website');
+      expect(allValues).toContain('Hello');
+      expect(allValues).toContain('messages');
+      expect(allValues).toContain('© 2024 Our Company');
+    });
+
+    it("should handle EJS templates with various tag types", async () => {
+      const input = `
+        <div>
+          <h2>User Dashboard</h2>
+          <% if (user.isAdmin) { %>
+            <p>Admin Panel</p>
+          <% } %>
+          <%# This is a comment %>
+          <p>Welcome back, <%- user.name %></p>
+          <span>Last login: <%= formatDate(user.lastLogin) %></span>
+        </div>
+      `;
+
+      const result = await loader.pull("en", input);
+      
+      expect(result).toHaveProperty("text_0");
+      expect(result).toHaveProperty("text_1");
+      expect(Object.keys(result).length).toBeGreaterThan(0);
+    });
+
+    it("should handle empty input", async () => {
+      const result = await loader.pull("en", "");
+      expect(result).toEqual({});
+    });
+
+    it("should handle input with only EJS tags", async () => {
+      const input = "<%= variable %><% if (condition) { %><% } %>";
+      const result = await loader.pull("en", input);
+      expect(result).toEqual({});
+    });
+
+    it("should handle mixed content", async () => {
+      const input = `
+        Welcome <%= user.name %>!
+        <% for (let i = 0; i < items.length; i++) { %>
+          Item: <%= items[i].name %>
+        <% } %>
+        Thank you for visiting.
+      `;
+
+      const result = await loader.pull("en", input);
+      expect(Object.keys(result).length).toBeGreaterThan(0);
+      expect(Object.values(result).some(text => text.includes("Welcome"))).toBe(true);
+      expect(Object.values(result).some(text => text.includes("Thank you"))).toBe(true);
+    });
+  });
+
+  describe("push", () => {
+    it("should reconstruct EJS template with translated content", async () => {
+      const originalInput = `<h1>Welcome</h1><p>Hello <%= name %></p>`;
+
+      // First pull to get the structure
+      const pulled = await loader.pull("en", originalInput);
+      
+      // Static translated data object based on actual loader behavior
+      const translated = {
+        text_0: "Bienvenido",
+        text_1: "Hola"
+      };
+
+      const result = await loader.push("es", translated);
+      
+      // Test against the expected reconstructed string
+      const expectedOutput = `<h1>Bienvenido</h1><p>Hola <%= name %></p>`;
+      
+      expect(result).toBe(expectedOutput);
+    });
+
+    it("should handle complex EJS templates", async () => {
+      const originalInput = `<h2>Dashboard</h2><% if (user) { %><p>Welcome</p><% } %>`;
+
+      const pulled = await loader.pull("en", originalInput);
+      
+      // Static translated data object 
+      const translated = {
+        text_0: "Tablero",
+        text_1: "Bienvenido"
+      };
+
+      const result = await loader.push("es", translated);
+      
+      // Test against the expected reconstructed string
+      const expectedOutput = `<h2>Tablero</h2><% if (user) { %><p>Bienvenido</p><% } %>`;
+      
+      expect(result).toBe(expectedOutput);
+    });
+
+    it("should handle missing original input", async () => {
+      const translated = {
+        text_0: "Hello World",
+        text_1: "This is a test"
+      };
+
+      const result = await loader.push("es", translated);
+      
+      expect(result).toContain("Hello World");
+      expect(result).toContain("This is a test");
+    });
+  });
+
+  describe("round trip", () => {
+    it("should maintain EJS functionality after round trip", async () => {
+      const originalInput = `
+        <h1>Welcome <%= title %></h1>
+        <% if (showMessage) { %>
+          <p>Hello <%= user.name %>, you have <%= count %> new messages.</p>
+        <% } %>
+        <ul>
+          <% items.forEach(function(item) { %>
+            <li><%= item.name %> - $<%= item.price %></li>
+          <% }); %>
+        </ul>
+        <footer>Contact us at [email protected]</footer>
+      `;
+
+      // Pull original content
+      const pulled = await loader.pull("en", originalInput);
+      
+      // Push back without translation (should be identical)
+      const reconstructed = await loader.push("en", pulled);
+      
+      // Verify EJS tags are preserved
+      expect(reconstructed).toContain("<%= title %>");
+      expect(reconstructed).toContain("<% if (showMessage) { %>");
+      expect(reconstructed).toContain("<%= user.name %>");
+      expect(reconstructed).toContain("<%= count %>");
+      expect(reconstructed).toContain("<% items.forEach(function(item) { %>");
+      expect(reconstructed).toContain("<%= item.name %>");
+      expect(reconstructed).toContain("<%= item.price %>");
+      expect(reconstructed).toContain("<% }); %>");
+      expect(reconstructed).toContain("Contact us at [email protected]");
+    });
+  });
+});

packages/cli/src/cli/loaders/ejs.ts

@@ -0,0 +1,167 @@
+import * as ejs from "ejs";
+import { ILoader } from "./_types";
+import { createLoader } from "./_utils";
+
+interface EjsParseResult {
+  content: string;
+  translatable: Record<string, string>;
+}
+
+function parseEjsForTranslation(input: string): EjsParseResult {
+  const translatable: Record<string, string> = {};
+  let counter = 0;
+
+  // Regular expression for all EJS tags
+  const ejsTagRegex = /<%[\s\S]*?%>/g;
+
+  // Split content by EJS tags, preserving both text and EJS parts
+  const parts: Array<{ type: 'text' | 'ejs', content: string }> = [];
+  let lastIndex = 0;
+  let match;
+
+  while ((match = ejsTagRegex.exec(input)) !== null) {
+    // Add text before the tag
+    if (match.index > lastIndex) {
+      parts.push({
+        type: 'text',
+        content: input.slice(lastIndex, match.index)
+      });
+    }
+    // Add the EJS tag
+    parts.push({
+      type: 'ejs',
+      content: match[0]
+    });
+    lastIndex = match.index + match[0].length;
+  }
+
+  // Add remaining text after the last tag
+  if (lastIndex < input.length) {
+    parts.push({
+      type: 'text',
+      content: input.slice(lastIndex)
+    });
+  }
+
+  // Build the template and extract translatable content
+  let template = '';
+  
+  for (const part of parts) {
+    if (part.type === 'ejs') {
+      // Keep EJS tags as-is
+      template += part.content;
+    } else {
+      // For text content, extract translatable parts while preserving HTML structure
+      const textContent = part.content;
+      
+      // Extract text content from HTML tags while preserving structure
+      const htmlTagRegex = /<[^>]+>/g;
+      const textParts: Array<{ type: 'html' | 'text', content: string }> = [];
+      let lastTextIndex = 0;
+      let htmlMatch;
+
+      while ((htmlMatch = htmlTagRegex.exec(textContent)) !== null) {
+        // Add text before the HTML tag
+        if (htmlMatch.index > lastTextIndex) {
+          const textBefore = textContent.slice(lastTextIndex, htmlMatch.index);
+          if (textBefore.trim()) {
+            textParts.push({ type: 'text', content: textBefore });
+          } else {
+            textParts.push({ type: 'html', content: textBefore });
+          }
+        }
+        // Add the HTML tag
+        textParts.push({ type: 'html', content: htmlMatch[0] });
+        lastTextIndex = htmlMatch.index + htmlMatch[0].length;
+      }
+
+      // Add remaining text after the last HTML tag
+      if (lastTextIndex < textContent.length) {
+        const remainingText = textContent.slice(lastTextIndex);
+        if (remainingText.trim()) {
+          textParts.push({ type: 'text', content: remainingText });
+        } else {
+          textParts.push({ type: 'html', content: remainingText });
+        }
+      }
+
+      // If no HTML tags found, treat entire content as text
+      if (textParts.length === 0) {
+        const trimmedContent = textContent.trim();
+        if (trimmedContent) {
+          textParts.push({ type: 'text', content: textContent });
+        } else {
+          textParts.push({ type: 'html', content: textContent });
+        }
+      }
+
+      // Process text parts
+      for (const textPart of textParts) {
+        if (textPart.type === 'text') {
+          const trimmedContent = textPart.content.trim();
+          if (trimmedContent) {
+            const key = `text_${counter++}`;
+            translatable[key] = trimmedContent;
+            template += textPart.content.replace(trimmedContent, `__LINGO_PLACEHOLDER_${key}__`);
+          } else {
+            template += textPart.content;
+          }
+        } else {
+          template += textPart.content;
+        }
+      }
+    }
+  }
+
+  return { content: template, translatable };
+}
+
+function reconstructEjsWithTranslation(template: string, translatable: Record<string, string>): string {
+  let result = template;
+  
+  // Replace placeholders with translated content
+  for (const [key, value] of Object.entries(translatable)) {
+    const placeholder = `__LINGO_PLACEHOLDER_${key}__`;
+    result = result.replace(new RegExp(placeholder, 'g'), value);
+  }
+  
+  return result;
+}
+
+export default function createEjsLoader(): ILoader<string, Record<string, any>> {
+  return createLoader({
+    async pull(locale, input) {
+      if (!input || input.trim() === '') {
+        return {};
+      }
+
+      try {
+        const parseResult = parseEjsForTranslation(input);
+        return parseResult.translatable;
+      } catch (error) {
+        console.warn('Warning: Could not parse EJS template, treating as plain text');
+        // Fallback: treat entire input as translatable content
+        return { content: input.trim() };
+      }
+    },
+
+    async push(locale, data, originalInput) {
+      if (!originalInput) {
+        // If no original input, reconstruct from data
+        return Object.values(data).join('\n');
+      }
+
+      try {
+        const parseResult = parseEjsForTranslation(originalInput);
+        
+        // Merge original translatable content with new translations
+        const mergedTranslatable = { ...parseResult.translatable, ...data };
+        
+        return reconstructEjsWithTranslation(parseResult.content, mergedTranslatable);
+      } catch (error) {
+        console.warn('Warning: Could not reconstruct EJS template, returning translated data');
+        return Object.values(data).join('\n');
+      }
+    },
+  });
+}