Merge pull request #90 from hydropix/dev

fix other languages field

Author: hydropix

benchmark/cli.py

@@ -454,19 +454,16 @@ def cmd_wiki_publish(args: argparse.Namespace) -> int:
         # Step 3: Copy generated files to wiki repo
         print(colored("Step 3/4: Copying files to wiki repository...", Colors.CYAN))
 
-        # Copy all markdown files
+        # Remove old subdirectories (now using flat structure)
+        for old_subdir in ["languages", "models"]:
+            old_dir = wiki_clone_dir / old_subdir
+            if old_dir.exists():
+                shutil.rmtree(old_dir)
+
+        # Copy all markdown files (flat structure)
         for md_file in wiki_output_dir.glob("*.md"):
             shutil.copy2(md_file, wiki_clone_dir / md_file.name)
 
-        # Copy subdirectories (languages, models)
-        for subdir in ["languages", "models"]:
-            src_dir = wiki_output_dir / subdir
-            dst_dir = wiki_clone_dir / subdir
-            if src_dir.exists():
-                if dst_dir.exists():
-                    shutil.rmtree(dst_dir)
-                shutil.copytree(src_dir, dst_dir)
-
         print(colored("Files copied.", Colors.GREEN))
 
         # Step 4: Commit and push

benchmark/translator.py

@@ -101,22 +101,27 @@ def _build_prompt(
         """
         system_prompt = f"""You are a professional {target_language} translator and writer.
 
-# CRITICAL: TARGET LANGUAGE IS {target_language.upper()}
+# TRANSLATION PRINCIPLES
 
-**YOUR TRANSLATION MUST BE WRITTEN ENTIRELY IN {target_language.upper()}.**
+Translate {source_language} to {target_language}. Output only the translation.
 
-You are translating FROM {source_language} TO {target_language}.
-Your output must be in {target_language} ONLY - do NOT use any other language.
+**PRIORITY ORDER:**
+1. Preserve exact names
+2. Match original tone and formality
+3. Use natural {target_language} phrasing - never word-for-word
+4. Fix grammar/spelling errors in output
+5. Translate idioms to {target_language} equivalents
+6. Preserve the author's literary style and emotional impact
 
-# TRANSLATION PRINCIPLES
+**QUALITY CHECK:**
+- Does it sound natural to a native {target_language} speaker?
+- Are all details from the original included?
+- Does punctuation follow {target_language} conventions?
 
-**Quality Standards:**
-- Translate faithfully while preserving the author's literary style, tone, and voice
-- Maintain the original meaning
-- Restructure sentences naturally in {target_language} (avoid word-by-word translation)
-- Adapt cultural references, idioms, and expressions to {target_language} context
-- Keep period-appropriate language when translating historical or classical texts
-- Preserve the emotional impact and atmosphere of the original
+If unsure between literal and natural phrasing: **choose natural**.
+
+**LAYOUT PRESERVATION:**
+- Keep the exact text layout, spacing, line breaks, and indentation
 - **WRITE YOUR TRANSLATION IN {target_language.upper()} - THIS IS MANDATORY**
 
 # FINAL REMINDER: YOUR OUTPUT LANGUAGE
@@ -134,12 +139,14 @@ def _build_prompt(
 4. Do NOT add explanations, comments, notes, or greetings
 
 **INCORRECT examples (DO NOT do this):**
-- "Here is the translation: {TRANSLATE_TAG_IN}Text...{TRANSLATE_TAG_OUT}"
-- "{TRANSLATE_TAG_IN}Text...{TRANSLATE_TAG_OUT} (Additional comment)"
-- "Sure! {TRANSLATE_TAG_IN}Text...{TRANSLATE_TAG_OUT}"
+❌ "Here is the translation: {TRANSLATE_TAG_IN}Text...{TRANSLATE_TAG_OUT}"
+❌ "{TRANSLATE_TAG_IN}Text...{TRANSLATE_TAG_OUT} (Additional comment)"
+❌ "Sure! {TRANSLATE_TAG_IN}Text...{TRANSLATE_TAG_OUT}"
+❌ "Text..." (missing tags entirely)
+❌ "{TRANSLATE_TAG_IN}Text..." (missing closing tag)
 
 **CORRECT format (ONLY this):**
-{TRANSLATE_TAG_IN}
+✅ {TRANSLATE_TAG_IN}
 Your translated text here
 {TRANSLATE_TAG_OUT}"""
 

benchmark/wiki/generator.py

@@ -152,13 +152,21 @@ def _get_language_info(self, code: str) -> dict:
         }
 
     def _slugify(self, text: str) -> str:
-        """Convert text to URL-safe slug."""
+        """Convert text to URL-safe slug for GitHub wiki page names."""
         slug = text.lower()
         slug = re.sub(r"[^a-z0-9\-_]", "-", slug)
         slug = re.sub(r"-+", "-", slug)
         slug = slug.strip("-")
         return slug
 
+    def _language_page_name(self, language_name: str) -> str:
+        """Generate wiki page name for a language (flat structure for GitHub wiki)."""
+        return f"Language-{self._slugify(language_name)}"
+
+    def _model_page_name(self, model_name: str) -> str:
+        """Generate wiki page name for a model (flat structure for GitHub wiki)."""
+        return f"Model-{self._slugify(model_name)}"
+
     def _calculate_score_distribution(self, scores: list[float]) -> dict:
         """Calculate score distribution buckets."""
         dist = {"excellent": 0, "good": 0, "acceptable": 0, "poor": 0, "failed": 0}
@@ -194,10 +202,8 @@ def generate_all(self, run_id: Optional[str] = None) -> Path:
         if run is None:
             raise ValueError("No benchmark run found")
 
-        # Ensure output directories exist
+        # Ensure output directory exists (flat structure for GitHub wiki)
         self.output_dir.mkdir(parents=True, exist_ok=True)
-        (self.output_dir / "languages").mkdir(exist_ok=True)
-        (self.output_dir / "models").mkdir(exist_ok=True)
 
         # Generate all pages
         self._generate_home(run)
@@ -218,7 +224,7 @@ def _generate_home(self, run: BenchmarkRun) -> None:
         for stats in sorted(model_stats, key=lambda x: x.avg_overall, reverse=True):
             model_rankings.append({
                 "name": stats.model,
-                "slug": self._slugify(stats.model),
+                "page_name": self._model_page_name(stats.model),
                 "avg_overall": stats.avg_overall,
                 "avg_accuracy": stats.avg_accuracy,
                 "avg_fluency": stats.avg_fluency,
@@ -235,7 +241,7 @@ def _generate_home(self, run: BenchmarkRun) -> None:
             language_rankings.append({
                 "name": lang_info["name"],
                 "native_name": lang_info["native_name"],
-                "slug": self._slugify(lang_info["name"]),
+                "page_name": self._language_page_name(lang_info["name"]),
                 "avg_overall": stats.avg_overall,
                 "indicator": get_score_indicator(stats.avg_overall),
                 "best_model": stats.best_model or "N/A",
@@ -269,16 +275,17 @@ def _generate_all_languages_page(self, run: BenchmarkRun) -> None:
         for stats in sorted(language_stats, key=lambda x: x.avg_overall, reverse=True):
             lang_info = self._get_language_info(stats.language_code)
             indicator = get_score_indicator(stats.avg_overall)
+            page_name = self._language_page_name(lang_info['name'])
             rows.append([
-                f"[{lang_info['name']}](languages/{self._slugify(lang_info['name'])})",
+                f"[{lang_info['name']}]({page_name})",
                 lang_info['native_name'],
                 lang_info['category'],
                 f"{indicator} {stats.avg_overall:.1f}",
                 stats.best_model or "N/A",
             ])
 
         table = format_markdown_table(headers, rows)
-        content = f"# All Languages\n\n{table}\n\n---\n\n[< Back to Home](Home)\n"
+        content = f"# All Languages\n\n{table}\n\n---\n\n[← Back to Home](Home)\n"
 
         (self.output_dir / "All-Languages.md").write_text(content, encoding="utf-8")
 
@@ -291,8 +298,9 @@ def _generate_all_models_page(self, run: BenchmarkRun) -> None:
 
         for stats in sorted(model_stats, key=lambda x: x.avg_overall, reverse=True):
             indicator = get_score_indicator(stats.avg_overall)
+            page_name = self._model_page_name(stats.model)
             rows.append([
-                f"[{stats.model}](models/{self._slugify(stats.model)})",
+                f"[{stats.model}]({page_name})",
                 f"{indicator} {stats.avg_overall:.1f}",
                 f"{stats.avg_accuracy:.1f}",
                 f"{stats.avg_fluency:.1f}",
@@ -301,7 +309,7 @@ def _generate_all_models_page(self, run: BenchmarkRun) -> None:
             ])
 
         table = format_markdown_table(headers, rows)
-        content = f"# All Models\n\n{table}\n\n---\n\n[< Back to Home](Home)\n"
+        content = f"# All Models\n\n{table}\n\n---\n\n[← Back to Home](Home)\n"
 
         (self.output_dir / "All-Models.md").write_text(content, encoding="utf-8")
 
@@ -340,7 +348,7 @@ def _generate_language_pages(self, run: BenchmarkRun) -> None:
                 if m_scores:
                     model_results.append({
                         "model": model,
-                        "model_slug": self._slugify(model),
+                        "model_page_name": self._model_page_name(model),
                         "avg_overall": sum(s.overall for s in m_scores) / len(m_scores),
                         "avg_accuracy": sum(s.accuracy for s in m_scores) / len(m_scores),
                         "avg_fluency": sum(s.fluency for s in m_scores) / len(m_scores),
@@ -367,16 +375,17 @@ def _generate_language_pages(self, run: BenchmarkRun) -> None:
                 total_translations=len(results),
                 model_results=model_results,
                 best_model=best_model,
-                best_model_slug=self._slugify(best_model),
+                best_model_page_name=self._model_page_name(best_model),
                 worst_model=worst_model,
-                worst_model_slug=self._slugify(worst_model),
+                worst_model_page_name=self._model_page_name(worst_model),
                 examples=examples,
                 score_dist=self._calculate_score_distribution(scores),
                 indicators=self.INDICATORS,
             )
 
-            filename = f"{self._slugify(lang_info['name'])}.md"
-            (self.output_dir / "languages" / filename).write_text(content, encoding="utf-8")
+            # Write to flat directory structure (GitHub wiki doesn't support subdirectories)
+            filename = f"{self._language_page_name(lang_info['name'])}.md"
+            (self.output_dir / filename).write_text(content, encoding="utf-8")
 
     def _generate_model_pages(self, run: BenchmarkRun) -> None:
         """Generate individual model pages."""
@@ -414,7 +423,7 @@ def _generate_model_pages(self, run: BenchmarkRun) -> None:
                     language_results.append({
                         "code": lang_code,
                         "name": lang_info["name"],
-                        "slug": self._slugify(lang_info["name"]),
+                        "page_name": self._language_page_name(lang_info["name"]),
                         "category": lang_info["category"],
                         "avg_overall": lang_avg,
                         "avg_accuracy": sum(s.accuracy for s in l_scores) / len(l_scores),
@@ -453,10 +462,10 @@ def _generate_model_pages(self, run: BenchmarkRun) -> None:
                 language_results=language_results,
                 categories=categories,
                 best_language=best_lang["name"] if best_lang else "N/A",
-                best_language_slug=self._slugify(best_lang["name"]) if best_lang else "",
+                best_language_page_name=self._language_page_name(best_lang["name"]) if best_lang else "",
                 best_language_score=best_lang["avg_overall"] if best_lang else 0,
                 worst_language=worst_lang["name"] if worst_lang else "N/A",
-                worst_language_slug=self._slugify(worst_lang["name"]) if worst_lang else "",
+                worst_language_page_name=self._language_page_name(worst_lang["name"]) if worst_lang else "",
                 worst_language_score=worst_lang["avg_overall"] if worst_lang else 0,
                 best_example=best_example,
                 worst_example=worst_example,
@@ -465,8 +474,9 @@ def _generate_model_pages(self, run: BenchmarkRun) -> None:
                 avg_translation_time_ms=avg_translation_time,
             )
 
-            filename = f"{self._slugify(model_name)}.md"
-            (self.output_dir / "models" / filename).write_text(content, encoding="utf-8")
+            # Write to flat directory structure (GitHub wiki doesn't support subdirectories)
+            filename = f"{self._model_page_name(model_name)}.md"
+            (self.output_dir / filename).write_text(content, encoding="utf-8")
 
     def _group_languages_by_category(self, language_rankings: list[dict]) -> list[dict]:
         """Group language rankings by category."""

benchmark/wiki/templates/home.md.j2

@@ -23,7 +23,7 @@ Overall performance across all tested languages:
 | Rank | Model | Avg Score | Accuracy | Fluency | Style | Languages Tested |
 |------|-------|-----------|----------|---------|-------|------------------|
 {% for model in model_rankings %}
-| {{ loop.index }} | [{{ model.name }}](models/{{ model.slug }}) | {{ model.indicator }} {{ model.avg_overall|round(1) }} | {{ model.avg_accuracy|round(1) }} | {{ model.avg_fluency|round(1) }} | {{ model.avg_style|round(1) }} | {{ model.languages_tested }} |
+| {{ loop.index }} | [{{ model.name }}]({{ model.page_name }}) | {{ model.indicator }} {{ model.avg_overall|round(1) }} | {{ model.avg_accuracy|round(1) }} | {{ model.avg_fluency|round(1) }} | {{ model.avg_style|round(1) }} | {{ model.languages_tested }} |
 {% endfor %}
 
 ---
@@ -35,7 +35,7 @@ Best translation quality by target language:
 | Rank | Language | Native | Avg Score | Best Model | Tests |
 |------|----------|--------|-----------|------------|-------|
 {% for lang in language_rankings[:15] %}
-| {{ loop.index }} | [{{ lang.name }}](languages/{{ lang.slug }}) | {{ lang.native_name }} | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.best_model }} | {{ lang.total_translations }} |
+| {{ loop.index }} | [{{ lang.name }}]({{ lang.page_name }}) | {{ lang.native_name }} | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.best_model }} | {{ lang.total_translations }} |
 {% endfor %}
 
 {% if language_rankings|length > 15 %}
@@ -64,7 +64,7 @@ Best translation quality by target language:
 | Language | Avg Score | Best Model |
 |----------|-----------|------------|
 {% for lang in category.languages %}
-| [{{ lang.name }}](languages/{{ lang.slug }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.best_model }} |
+| [{{ lang.name }}]({{ lang.page_name }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.best_model }} |
 {% endfor %}
 
 {% endfor %}

benchmark/wiki/templates/language.md.j2

@@ -15,8 +15,8 @@
 | **Fluency** | {{ avg_fluency|round(1) }}/10 |
 | **Style** | {{ avg_style|round(1) }}/10 |
 | **Total Tests** | {{ total_translations }} |
-| **Best Model** | [{{ best_model }}](../models/{{ best_model_slug }}) |
-| **Worst Model** | [{{ worst_model }}](../models/{{ worst_model_slug }}) |
+| **Best Model** | [{{ best_model }}]({{ best_model_page_name }}) |
+| **Worst Model** | [{{ worst_model }}]({{ worst_model_page_name }}) |
 
 ---
 
@@ -25,7 +25,7 @@
 | Model | Overall | Accuracy | Fluency | Style |
 |-------|---------|----------|---------|-------|
 {% for result in model_results %}
-| [{{ result.model }}](../models/{{ result.model_slug }}) | {{ result.indicator }} {{ result.avg_overall|round(1) }} | {{ result.avg_accuracy|round(1) }} | {{ result.avg_fluency|round(1) }} | {{ result.avg_style|round(1) }} |
+| [{{ result.model }}]({{ result.model_page_name }}) | {{ result.indicator }} {{ result.avg_overall|round(1) }} | {{ result.avg_accuracy|round(1) }} | {{ result.avg_fluency|round(1) }} | {{ result.avg_style|round(1) }} |
 {% endfor %}
 
 ---
@@ -70,4 +70,4 @@
 
 ---
 
-[< Back to Home](../Home) | [All Languages](../All-Languages)
+[← Back to Home](Home) | [All Languages](All-Languages)

benchmark/wiki/templates/model.md.j2

@@ -14,8 +14,8 @@
 | **Style** | {{ avg_style|round(1) }}/10 |
 | **Languages Tested** | {{ total_languages }} |
 | **Total Translations** | {{ total_translations }} |
-| **Best Language** | [{{ best_language }}](../languages/{{ best_language_slug }}) ({{ best_language_score|round(1) }}) |
-| **Worst Language** | [{{ worst_language }}](../languages/{{ worst_language_slug }}) ({{ worst_language_score|round(1) }}) |
+| **Best Language** | [{{ best_language }}]({{ best_language_page_name }}) ({{ best_language_score|round(1) }}) |
+| **Worst Language** | [{{ worst_language }}]({{ worst_language_page_name }}) ({{ worst_language_score|round(1) }}) |
 
 ---
 
@@ -26,7 +26,7 @@
 | Rank | Language | Overall | Accuracy | Fluency | Style |
 |------|----------|---------|----------|---------|-------|
 {% for lang in language_results[:10] %}
-| {{ loop.index }} | [{{ lang.name }}](../languages/{{ lang.slug }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.avg_accuracy|round(1) }} | {{ lang.avg_fluency|round(1) }} | {{ lang.avg_style|round(1) }} |
+| {{ loop.index }} | [{{ lang.name }}]({{ lang.page_name }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.avg_accuracy|round(1) }} | {{ lang.avg_fluency|round(1) }} | {{ lang.avg_style|round(1) }} |
 {% endfor %}
 
 {% if language_results|length > 10 %}
@@ -36,7 +36,7 @@
 | Rank | Language | Overall | Accuracy | Fluency | Style |
 |------|----------|---------|----------|---------|-------|
 {% for lang in language_results %}
-| {{ loop.index }} | [{{ lang.name }}](../languages/{{ lang.slug }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.avg_accuracy|round(1) }} | {{ lang.avg_fluency|round(1) }} | {{ lang.avg_style|round(1) }} |
+| {{ loop.index }} | [{{ lang.name }}]({{ lang.page_name }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.avg_accuracy|round(1) }} | {{ lang.avg_fluency|round(1) }} | {{ lang.avg_style|round(1) }} |
 {% endfor %}
 
 </details>
@@ -52,7 +52,7 @@
 | Language | Overall | Accuracy | Fluency | Style |
 |----------|---------|----------|---------|-------|
 {% for lang in category.languages %}
-| [{{ lang.name }}](../languages/{{ lang.slug }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.avg_accuracy|round(1) }} | {{ lang.avg_fluency|round(1) }} | {{ lang.avg_style|round(1) }} |
+| [{{ lang.name }}]({{ lang.page_name }}) | {{ lang.indicator }} {{ lang.avg_overall|round(1) }} | {{ lang.avg_accuracy|round(1) }} | {{ lang.avg_fluency|round(1) }} | {{ lang.avg_style|round(1) }} |
 {% endfor %}
 
 **Category Average:** {{ category.indicator }} {{ category.avg_overall|round(1) }}
@@ -130,4 +130,4 @@
 
 ---
 
-[< Back to Home](../Home) | [All Models](../All-Models)
+[← Back to Home](Home) | [All Models](All-Models)