Upload all models and assets for bar (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: bar
-language_name: BAR
+language_name: Bavarian
 language_family: germanic_west_continental
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-germanic_west_continental
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.002
+    value: 4.003
   - name: best_isotropy
     type: isotropy
-    value: 0.8442
+    value: 0.8432
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# BAR - Wikilangs Models
+# Bavarian - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **BAR** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Bavarian** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.167x | 3.17 | 0.0429% | 1,049,729 |
-| **16k** | 3.475x | 3.48 | 0.0470% | 956,699 |
-| **32k** | 3.752x | 3.75 | 0.0508% | 885,998 |
-| **64k** | 4.002x 🏆 | 4.00 | 0.0542% | 830,614 |
+| **8k** | 3.167x | 3.17 | 0.0430% | 1,042,115 |
+| **16k** | 3.477x | 3.48 | 0.0472% | 949,394 |
+| **32k** | 3.753x | 3.75 | 0.0509% | 879,530 |
+| **64k** | 4.003x 🏆 | 4.00 | 0.0543% | 824,531 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Buffalo County Obgruafa am 22. Feba is a County in Wisconsin in da USA. Beleg Im...`
+**Sample 1:** `Forstern is a Gmoa im obaboarischn Landkroas Arrdeng. Im Netz Gemeinde Forstern ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁buffalo ▁county ▁obgruafa ▁am ▁ 2 2 . ▁feba ▁is ... (+13 more)` | 23 |
-| 16k | `▁buffalo ▁county ▁obgruafa ▁am ▁ 2 2 . ▁feba ▁is ... (+13 more)` | 23 |
-| 32k | `▁buffalo ▁county ▁obgruafa ▁am ▁ 2 2 . ▁feba ▁is ... (+13 more)` | 23 |
-| 64k | `▁buffalo ▁county ▁obgruafa ▁am ▁ 2 2 . ▁feba ▁is ... (+13 more)` | 23 |
+| 8k | `▁forst ern ▁is ▁a ▁gmoa ▁im ▁oba boarischn ▁landkroas ▁ar ... (+19 more)` | 29 |
+| 16k | `▁forst ern ▁is ▁a ▁gmoa ▁im ▁obaboarischn ▁landkroas ▁arrdeng . ... (+15 more)` | 25 |
+| 32k | `▁forst ern ▁is ▁a ▁gmoa ▁im ▁obaboarischn ▁landkroas ▁arrdeng . ... (+13 more)` | 23 |
+| 64k | `▁forst ern ▁is ▁a ▁gmoa ▁im ▁obaboarischn ▁landkroas ▁arrdeng . ... (+12 more)` | 22 |
 
-**Sample 2:** `Fauquier County. Obgruafa am 22. Feba is a County in Virginia in da USA. Beleg I...`
+**Sample 2:** `Marlboro County. Obgruafa am 22. Feba is a County in South Carolina in da USA. B...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁f au qui er ▁county . ▁obgruafa ▁am ▁ 2 ... (+17 more)` | 27 |
-| 16k | `▁f au qui er ▁county . ▁obgruafa ▁am ▁ 2 ... (+17 more)` | 27 |
-| 32k | `▁fau qui er ▁county . ▁obgruafa ▁am ▁ 2 2 ... (+16 more)` | 26 |
-| 64k | `▁fau qui er ▁county . ▁obgruafa ▁am ▁ 2 2 ... (+16 more)` | 26 |
+| 8k | `▁mar l boro ▁county . ▁obgruafa ▁am ▁ 2 2 ... (+18 more)` | 28 |
+| 16k | `▁mar l boro ▁county . ▁obgruafa ▁am ▁ 2 2 ... (+18 more)` | 28 |
+| 32k | `▁marl boro ▁county . ▁obgruafa ▁am ▁ 2 2 . ... (+17 more)` | 27 |
+| 64k | `▁marlboro ▁county . ▁obgruafa ▁am ▁ 2 2 . ▁feba ... (+16 more)` | 26 |
 
-**Sample 3:** `Carlow stähd fia: Carlow, Stod in Irland County Carlow, irische Grofschoft Carlo...`
+**Sample 3:** `Hill County is a County in Montana in da USA. Beleg Im Netz in Montana`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁carl ow ▁stähd ▁fia : ▁carl ow , ▁stod ▁in ... (+18 more)` | 28 |
-| 16k | `▁carl ow ▁stähd ▁fia : ▁carl ow , ▁stod ▁in ... (+17 more)` | 27 |
-| 32k | `▁carl ow ▁stähd ▁fia : ▁carl ow , ▁stod ▁in ... (+16 more)` | 26 |
-| 64k | `▁carlow ▁stähd ▁fia : ▁carlow , ▁stod ▁in ▁irland ▁county ... (+10 more)` | 20 |
+| 8k | `▁hill ▁county ▁is ▁a ▁county ▁in ▁montana ▁in ▁da ▁usa ... (+6 more)` | 16 |
+| 16k | `▁hill ▁county ▁is ▁a ▁county ▁in ▁montana ▁in ▁da ▁usa ... (+6 more)` | 16 |
+| 32k | `▁hill ▁county ▁is ▁a ▁county ▁in ▁montana ▁in ▁da ▁usa ... (+6 more)` | 16 |
+| 64k | `▁hill ▁county ▁is ▁a ▁county ▁in ▁montana ▁in ▁da ▁usa ... (+6 more)` | 16 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.002x compression
-- **Lowest UNK Rate:** 8k with 0.0429% unknown tokens
+- **Best Compression:** 64k achieves 4.003x compression
+- **Lowest UNK Rate:** 8k with 0.0430% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 27,417 | 14.74 | 110,635 | 12.9% | 31.4% |
-| **2-gram** | Subword | 362 🏆 | 8.50 | 7,805 | 60.7% | 98.3% |
-| **3-gram** | Word | 41,058 | 15.33 | 129,534 | 12.6% | 26.5% |
-| **3-gram** | Subword | 3,802 | 11.89 | 63,080 | 20.6% | 60.8% |
-| **4-gram** | Word | 57,367 | 15.81 | 187,348 | 13.7% | 25.1% |
-| **4-gram** | Subword | 27,463 | 14.75 | 363,810 | 9.1% | 28.4% |
+| **2-gram** | Word | 27,199 | 14.73 | 109,780 | 13.0% | 31.5% |
+| **2-gram** | Subword | 361 🏆 | 8.50 | 7,796 | 60.7% | 98.3% |
+| **3-gram** | Word | 40,782 | 15.32 | 128,747 | 12.7% | 26.6% |
+| **3-gram** | Subword | 3,796 | 11.89 | 62,893 | 20.6% | 60.9% |
+| **4-gram** | Word | 56,976 | 15.80 | 186,218 | 13.7% | 25.1% |
+| **4-gram** | Subword | 27,410 | 14.74 | 362,482 | 9.1% | 28.4% |
+| **5-gram** | Word | 38,882 | 15.25 | 130,277 | 15.7% | 28.0% |
+| **5-gram** | Subword | 124,788 | 16.93 | 1,153,187 | 4.9% | 16.5% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `vo da` | 26,665 |
-| 2 | `is a` | 22,998 |
-| 3 | `in da` | 22,567 |
-| 4 | `im netz` | 14,649 |
-| 5 | `vo de` | 13,503 |
+| 1 | `vo da` | 26,508 |
+| 2 | `is a` | 22,819 |
+| 3 | `in da` | 22,392 |
+| 4 | `im netz` | 14,484 |
+| 5 | `vo de` | 13,424 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `beleg im netz` | 3,527 |
+| 1 | `beleg im netz` | 3,530 |
 | 2 | `in da usa` | 3,478 |
 | 3 | `da beziak hod` | 2,393 |
-| 4 | `des is a` | 2,037 |
-| 5 | `im netz in` | 2,001 |
+| 4 | `im netz in` | 2,005 |
+| 5 | `sitz vo da` | 1,888 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `beleg im netz in` | 1,573 |
-| 2 | `da sitz vo da` | 1,483 |
+| 1 | `beleg im netz in` | 1,575 |
+| 2 | `da sitz vo da` | 1,482 |
 | 3 | `is a county in` | 1,429 |
 | 4 | `in da usa da` | 1,407 |
 | 5 | `a katastralgmoa in da` | 1,387 |
 
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `flächn ausgwiesn gwesn ende woarn` | 1,385 |
+| 2 | `hektar ois laundwiatschoftliche flächn gnutzt` | 1,385 |
+| 3 | `forstwirtschaftli gnutzte flächn ausgwiesn gwesn` | 1,385 |
+| 4 | `hektar sand ois forstwirtschaftli gnutzte` | 1,385 |
+| 5 | `ois laundwiatschoftliche flächn gnutzt und` | 1,385 |
+
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `n _` | 706,670 |
-| 2 | `a _` | 671,532 |
-| 3 | `c h` | 640,658 |
-| 4 | `_ d` | 560,830 |
-| 5 | `e _` | 482,452 |
+| 1 | `n _` | 701,951 |
+| 2 | `a _` | 667,528 |
+| 3 | `c h` | 636,525 |
+| 4 | `_ d` | 557,323 |
+| 5 | `e _` | 479,658 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `s c h` | 305,657 |
-| 2 | `_ d e` | 255,292 |
-| 3 | `_ d a` | 174,094 |
-| 4 | `n d _` | 170,331 |
-| 5 | `d a _` | 169,070 |
+| 1 | `s c h` | 303,728 |
+| 2 | `_ d e` | 253,515 |
+| 3 | `_ d a` | 172,902 |
+| 4 | `n d _` | 169,557 |
+| 5 | `u n d` | 168,298 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d a _` | 133,118 |
-| 2 | `_ d e _` | 131,138 |
-| 3 | `u n d _` | 128,509 |
-| 4 | `_ u n d` | 120,455 |
-| 5 | `i s c h` | 100,072 |
+| 1 | `_ d a _` | 132,086 |
+| 2 | `_ d e _` | 130,374 |
+| 3 | `u n d _` | 127,939 |
+| 4 | `_ u n d` | 119,950 |
+| 5 | `i s c h` | 99,379 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ u n d _` | 118,720 |
+| 2 | `_ v o _ d` | 44,559 |
+| 3 | `_ i n _ d` | 37,539 |
+| 4 | `i s c h e` | 33,643 |
+| 5 | `_ d e s _` | 31,011 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 362
+- **Best Perplexity:** 2-gram (subword) with 361
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~28% of corpus
+- **Coverage:** Top-1000 patterns cover ~17% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.7091 | 1.635 | 5.19 | 569,846 | 29.1% |
-| **1** | Subword | 0.9426 | 1.922 | 6.61 | 3,388 | 5.7% |
-| **2** | Word | 0.2116 | 1.158 | 1.52 | 2,948,968 | 78.8% |
-| **2** | Subword | 0.9158 | 1.887 | 5.85 | 22,382 | 8.4% |
-| **3** | Word | 0.0664 | 1.047 | 1.12 | 4,475,523 | 93.4% |
-| **3** | Subword | 0.8683 | 1.826 | 4.67 | 130,801 | 13.2% |
-| **4** | Word | 0.0224 🏆 | 1.016 | 1.04 | 4,973,907 | 97.8% |
-| **4** | Subword | 0.7777 | 1.714 | 3.53 | 610,360 | 22.2% |
+| **1** | Word | 0.7076 | 1.633 | 5.17 | 567,851 | 29.2% |
+| **1** | Subword | 0.9427 | 1.922 | 6.61 | 3,387 | 5.7% |
+| **2** | Word | 0.2111 | 1.158 | 1.52 | 2,930,161 | 78.9% |
+| **2** | Subword | 0.9146 | 1.885 | 5.83 | 22,370 | 8.5% |
+| **3** | Word | 0.0663 | 1.047 | 1.11 | 4,443,260 | 93.4% |
+| **3** | Subword | 0.8673 | 1.824 | 4.66 | 130,496 | 13.3% |
+| **4** | Word | 0.0224 🏆 | 1.016 | 1.04 | 4,937,652 | 97.8% |
+| **4** | Subword | 0.7772 | 1.714 | 3.53 | 608,299 | 22.3% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `de flugsauria buidns de knaj oda negakuss domois ghairat hod direkt in den jüngling oder goar`
-2. `da insl blaagad hom niks gwisst hod a öatschoft im netz hoamseitn vo mercia zrugg mei`
-3. `und is er im neich augleande mocha oda z himinbjörg und ů und san letztle zua`
+1. `de gepidn und bbö 178 bukit tinggi 72 canon triplex a 7 hz ws touro college`
+2. `da effentlichn stroßn am 9 verletzter blick af de gebietskeapaschoftn in bayern gwen dem meearesspia...`
+3. `und alfonso cuarón timothy j nö öbb infra öbb pv tullnerfelder bahn rengschbuach grünthal geografie ...`
 
 **Context Size 2:**
 
-1. `vo da vawoitung is in n gensatz za altn welt is a bleamalkiag ausgoat und in bemen`
-2. `is a urtschoft und a jeda miassat eintritt brandln dann war des eagebnis vo de großn industriezentre...`
-3. `in da usa on da anderson mesa in da langobardischn ehefrow vom kini ludwig i vo habsbuag`
+1. `vo da blaa oim aussa und entschdengan seine wichdigstn litararischn weak da voda vo da gmoa kirchham`
+2. `is a kuaza a1 kuaza mit klima b launga und zwoa enklkinda da hoeneß uli z bad`
+3. `in da katastralgmoa dobranberg zsammgrechnt 84 bauflächn mit 44 633 m und 58 gärten auf 135 526`
 
 **Context Size 3:**
 
-1. `in da usa beleg im netz in south carolina in da usa da beziak hod a fläche vo`
-2. `beleg im netz eana hoamseitn eana myspace seitn eana facebook seitn volksmusik`
-3. `da beziak hod a fläch vo 802 km af dena 49 970 eihwohna lem stond gmoana da powiat`
+1. `in da usa beleg im netz in virginia`
+2. `beleg im netz in missouri`
+3. `da beziak hod 39 451 eihwohna da sitz vo da vawoitung is leoti da beziak hod 12 786`
 
 **Context Size 4:**
 
-1. `beleg im netz in der normandie im département seine maritime in da region normandie ea liegd im arro...`
-2. `da sitz vo da vawoitung is lake city da beziak hod a flächn vo quadratkilometa dovo san 1 quadratkil...`
-3. `is a county in virginia in da usa beleg im netz in missouri`
+1. `beleg im netz in nebraska`
+2. `da sitz vo da kroasvawoitung vo oanign landkroas liegt außahoib vom landkroas oft in da namasgleichn...`
+3. `is a county in wisconsin in da usa beleg im netz in der emilia romagna des europapreises`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_heerinznachrnea`
-2. `an_knenant_getun`
-3. `eichaumo_bh_ll,_`
+1. `_w.adaiwenieurio`
+2. `a_lidovicröniser`
+3. `e_hmbrkum_runís_`
 
 **Context Size 2:**
 
-1. `n_de_autz)_val_(z`
-2. `a_berseeka)_trejn`
-3. `chulretiveicittbo`
+1. `n_fc_rein_wieforo`
+2. `a_da_oschofferkea`
+3. `chr_koi'seybunds_`
 
 **Context Size 3:**
 
-1. `sch-wei_in_de_im_o`
-2. `_der_schaubind_so_`
-3. `_da_hamation_phana`
+1. `schburyan_no_san_d`
+2. `_dem_scusdecentisc`
+3. `_daument_in_und_zu`
 
 **Context Size 4:**
 
-1. `_da_sicht_große_sog`
-2. `_de_kompillatinen_t`
-3. `und_europäischen_(a`
+1. `_da_letztn_de_ameri`
+2. `_de_marekd_om_auf_1`
+3. `und_botta_200+_maß_`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 97.8% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (610,360 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (608,299 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,48 +346,48 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 213,465 |
-| Total Tokens | 5,378,004 |
-| Mean Frequency | 25.19 |
+| Vocabulary Size | 212,365 |
+| Total Tokens | 5,339,853 |
+| Mean Frequency | 25.14 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 715.69 |
+| Frequency Std Dev | 712.67 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | de | 137,737 |
-| 2 | da | 137,316 |
-| 3 | und | 119,692 |
-| 4 | in | 102,651 |
-| 5 | a | 92,739 |
-| 6 | vo | 92,570 |
-| 7 | is | 86,950 |
-| 8 | im | 71,173 |
-| 9 | des | 34,457 |
-| 10 | hod | 30,772 |
+| 1 | de | 136,913 |
+| 2 | da | 136,168 |
+| 3 | und | 119,185 |
+| 4 | in | 101,699 |
+| 5 | a | 92,218 |
+| 6 | vo | 91,584 |
+| 7 | is | 86,664 |
+| 8 | im | 70,677 |
+| 9 | des | 33,854 |
+| 10 | hod | 30,719 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | vorarlberga | 2 |
-| 2 | opfenbach | 2 |
-| 3 | raubibafäi | 2 |
-| 4 | marcianopel | 2 |
-| 5 | sachtler | 2 |
-| 6 | vitec | 2 |
-| 7 | videocom | 2 |
-| 8 | promovierten | 2 |
-| 9 | mechanisches | 2 |
-| 10 | stabilisierungssystem | 2 |
+| 1 | mechanisches | 2 |
+| 2 | stabilisierungssystem | 2 |
+| 3 | voeffentlecht | 2 |
+| 4 | innpuls | 2 |
+| 5 | buagstej | 2 |
+| 6 | nuwenburg | 2 |
+| 7 | kulturweges | 2 |
+| 8 | spessartprojektes | 2 |
+| 9 | terrassnfermig | 2 |
+| 10 | tuamhigi | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9728 |
-| R² (Goodness of Fit) | 0.999432 |
+| Zipf Coefficient | 0.9730 |
+| R² (Goodness of Fit) | 0.999444 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
@@ -371,7 +403,7 @@ Below are text samples generated from each subword-based Markov chain model:
 
 - **Zipf Compliance:** R²=0.9994 indicates excellent adherence to Zipf's law
 - **High Frequency Dominance:** Top 100 words cover 34.1% of corpus
-- **Long Tail:** 203,465 words needed for remaining 23.3% coverage
+- **Long Tail:** 202,365 words needed for remaining 23.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8230 | 0.3300 | N/A | N/A |
-| **mono_64d** | 64 | 0.8442 🏆 | 0.2564 | N/A | N/A |
-| **mono_128d** | 128 | 0.8427 | 0.1773 | N/A | N/A |
+| **mono_32d** | 32 | 0.8296 | 0.3402 | N/A | N/A |
+| **mono_64d** | 64 | 0.8410 | 0.2581 | N/A | N/A |
+| **mono_128d** | 128 | 0.8432 🏆 | 0.1737 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8296 | 0.3341 | 0.0920 | 0.3960 |
+| **aligned_64d** | 64 | 0.8410 | 0.2543 | 0.1940 | 0.6020 |
+| **aligned_128d** | 128 | 0.8432 | 0.1862 | 0.2860 | 0.6780 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_64d with 0.8442 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2546. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_128d with 0.8432 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2578. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 28.6% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.694** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,17 +461,18 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-be` | begleitendn, bewohnde, bewiabt |
-| `-sc` | schwingt, schienengebundenen, schwefelhölzern |
+| `-sc` | scharmbeck, schitznvaein, schiaf |
+| `-sch` | scharmbeck, schitznvaein, schiaf |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-n` | begleitendn, lesegerätn, clipperton |
-| `-en` | warmgemäßigten, alanen, aussen |
-| `-er` | puppentheater, kirchenmusiker, rothmüller |
-| `-ng` | hamhŭng, polung, urauffüahrung |
-| `-ch` | woifschbouch, mittlboarisch, meafoch |
+| `-n` | şabran, unterwestern, weidesdn |
+| `-en` | metallen, theologen, münzen |
+| `-ng` | wondering, pisang, umwondlung |
+| `-er` | gräberfelder, eichenauer, weydenhammer |
+| `-ch` | hoierschbouch, weißabgleich, obergreutschach |
+| `-ung` | umwondlung, auflösung, ausbroadung |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -444,18 +480,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `icht` | 1.84x | 346 contexts | richt, eicht, dicht |
-| `schr` | 2.11x | 137 contexts | schrei, schräg, schrag |
-| `gsch` | 1.93x | 181 contexts | gschdö, gscher, gschod |
-| `schl` | 1.64x | 288 contexts | eschl, ischl, göschl |
-| `chte` | 1.70x | 217 contexts | åchte, echte, ochte |
-| `itsc` | 2.10x | 64 contexts | gitsch, kitsch, nitsch |
-| `chof` | 2.22x | 50 contexts | schof, schofn, schoft |
-| `tlic` | 1.76x | 137 contexts | etlich, etlichs, rötlich |
-| `atio` | 2.18x | 45 contexts | natio, ratio, nation |
-| `nisc` | 1.72x | 127 contexts | nisch, nischt, nischn |
-| `ichn` | 1.97x | 68 contexts | eichn, suichn, zoichn |
-| `uach` | 1.78x | 99 contexts | duach, buach, suach |
+| `ster` | 2.00x | 209 contexts | aster, ester, stern |
+| `schl` | 1.77x | 287 contexts | eschl, ischl, schlau |
+| `schr` | 1.99x | 137 contexts | schrit, schrim, schreg |
+| `gsch` | 1.77x | 181 contexts | gschai, gschdö, gschmo |
+| `uach` | 1.99x | 99 contexts | buach, huach, suach |
+| `itsc` | 2.19x | 64 contexts | gitsch, nitsch, kitsch |
+| `icht` | 1.54x | 345 contexts | eicht, wicht, richt |
+| `atio` | 2.26x | 45 contexts | ratio, natio, nation |
+| `nisc` | 1.77x | 126 contexts | nisch, nischn, nischt |
+| `reic` | 1.78x | 97 contexts | reich, reichd, reichl |
+| `chof` | 2.07x | 50 contexts | schof, schoft, schofn |
+| `tion` | 1.73x | 93 contexts | tione, aktion, notion |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -463,16 +499,12 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-sc` | `-n` | 53 words | schleierbaracken, schüidln |
-| `-be` | `-n` | 43 words | betroffanan, berichtigungen |
-| `-sc` | `-en` | 13 words | schleierbaracken, schnupfen |
-| `-be` | `-ng` | 13 words | bevejkarungsentwigglung, bereicherung |
-| `-sc` | `-er` | 13 words | schimpfkalender, schweller |
-| `-sc` | `-ch` | 10 words | schrambach, schpruch |
-| `-be` | `-en` | 9 words | berichtigungen, beten |
-| `-be` | `-ch` | 4 words | besuch, bessenbach |
-| `-be` | `-er` | 3 words | bettinger, berghammer |
-| `-sc` | `-ng` | 3 words | schoidruckpeglmindarung, schiefling |
+| `-sc` | `-n` | 52 words | schbondan, schbüün |
+| `-sc` | `-er` | 16 words | schatzgräber, schweinsteiger |
+| `-sc` | `-en` | 13 words | schlampen, screven |
+| `-sc` | `-ng` | 11 words | schädlbedeckung, schraubvabindung |
+| `-sc` | `-ch` | 10 words | scharlach, schbruch |
+| `-sc` | `-ung` | 4 words | schädlbedeckung, schraubvabindung |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -480,26 +512,28 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| betreiber | **`be-treib-er`** | 6.0 | `treib` |
-| vorarlberger | **`vorarlberg-er`** | 4.5 | `vorarlberg` |
-| verkaufen | **`verkauf-en`** | 4.5 | `verkauf` |
-| grotesken | **`grotesk-en`** | 4.5 | `grotesk` |
-| schwabinger | **`sc-hwabi-ng-er`** | 4.5 | `hwabi` |
-| gsprochenen | **`gspro-ch-en-en`** | 4.5 | `gspro` |
-| waxenberger | **`waxenberg-er`** | 4.5 | `waxenberg` |
-| scheazhoft | **`sc-heazhoft`** | 4.5 | `heazhoft` |
-| gebildeten | **`gebildet-en`** | 4.5 | `gebildet` |
-| carstensen | **`carstens-en`** | 4.5 | `carstens` |
-| bewundern | **`be-wundern`** | 4.5 | `wundern` |
-| dornröschen | **`dornrös-ch-en`** | 3.0 | `dornrös` |
-| überetscher | **`überets-ch-er`** | 3.0 | `überets` |
-| betrieblich | **`be-triebli-ch`** | 3.0 | `triebli` |
-| umgebungen | **`umgebu-ng-en`** | 3.0 | `umgebu` |
+| schnitzen | **`sch-nitz-en`** | 6.0 | `nitz` |
+| enthaltenen | **`enthalt-en-en`** | 6.0 | `enthalt` |
+| schwensen | **`sch-wens-en`** | 6.0 | `wens` |
+| herrnhausen | **`herrnhaus-en`** | 4.5 | `herrnhaus` |
+| schrottenberg | **`sch-rottenberg`** | 4.5 | `rottenberg` |
+| heaschafamülien | **`heaschafamüli-en`** | 4.5 | `heaschafamüli` |
+| fawoitung | **`fawoit-ung`** | 4.5 | `fawoit` |
+| regulären | **`regulär-en`** | 4.5 | `regulär` |
+| leitmeritzer | **`leitmeritz-er`** | 4.5 | `leitmeritz` |
+| jungfrauen | **`jungfrau-en`** | 4.5 | `jungfrau` |
+| gespenster | **`gespenst-er`** | 4.5 | `gespenst` |
+| dynastien | **`dynasti-en`** | 4.5 | `dynasti` |
+| referenten | **`referent-en`** | 4.5 | `referent` |
+| birkenhainer | **`birkenhain-er`** | 4.5 | `birkenhain` |
+| rettersheimer | **`rettersheim-er`** | 4.5 | `rettersheim` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language BAR appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Bavarian shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -511,7 +545,7 @@ The language BAR appears to be more isolating or has a highly fixed vocabulary.
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
 | Tokenizer | **64k BPE** | Best compression (4.00x) |
-| N-gram | **2-gram** | Lowest perplexity (362) |
+| N-gram | **2-gram** | Lowest perplexity (361) |
 | Markov | **Context-4** | Highest predictability (97.8%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -726,4 +760,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 06:42:51*
+*Report Date: 2026-01-03 19:01:37*

models/embeddings/aligned/bar_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c79fb944f3fa50595a99205e89ad8c54fbd5f7e783aa26c55e502f8dabba673
+size 1115259088

models/embeddings/aligned/bar_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "bar", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bar_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ca1ac465d2dd95cc16e424657eeb7db432ce017a4efb8aee3caa34f9d6738a1
+size 65664

models/embeddings/aligned/bar_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bar",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 33406,
+  "vocab_size": 87616
+}

models/embeddings/aligned/bar_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74f527750a81c14b7fc7a4ab52eb0da0010ae6baf877edd04025787c141156ee
+size 279970000

models/embeddings/aligned/bar_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "bar", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bar_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:369418a6de4a717f66d0996acb8ace1729ec23b868acdd72d960a800c594e096
+size 4224

models/embeddings/aligned/bar_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bar",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 33406,
+  "vocab_size": 87616
+}

models/embeddings/aligned/bar_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b201763425b770374e8a2936ad509eba2f621f422ff3cd6bf0c510b5b885e7b
+size 558399696

models/embeddings/aligned/bar_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "bar", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/bar_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68ddbc4bb448934021ab28cd8694eff921ab731ed291b72068994be8e18873e
+size 16512

models/embeddings/aligned/bar_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "bar",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 33406,
+  "vocab_size": 87616
+}

models/embeddings/monolingual/bar_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6256ddf046563a10ad0e96ef58deffac4839b1eb48bc65facd0dd74ba05e830b
-size 1115788582
+oid sha256:0c79fb944f3fa50595a99205e89ad8c54fbd5f7e783aa26c55e502f8dabba673
+size 1115259088

models/embeddings/monolingual/bar_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 88124
+  "vocab_size": 87616
 }

models/embeddings/monolingual/bar_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc7a6c48e154f63a623a8376127a01b222af8bf2a04aba41f602a929eb6c27af
-size 280109350
+oid sha256:74f527750a81c14b7fc7a4ab52eb0da0010ae6baf877edd04025787c141156ee
+size 279970000

models/embeddings/monolingual/bar_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 88124
+  "vocab_size": 87616
 }

models/embeddings/monolingual/bar_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9998545a04e63264d36cca0eb51c1e069804e9cdba44f02039cd26cd05531df8
-size 558669094
+oid sha256:6b201763425b770374e8a2936ad509eba2f621f422ff3cd6bf0c510b5b885e7b
+size 558399696

models/embeddings/monolingual/bar_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 88124
+  "vocab_size": 87616
 }

models/subword_markov/bar_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60e6c3a72c3c4d893f6ddf06c56e1234d3475b1b1443e819143470ab5f1b4e23
-size 182402
+oid sha256:4bd0b0ccb19f4c10f880f9830d26c9bb55bab36620ceb93df4a84adf0ef78538
+size 179926

models/subword_markov/bar_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "bar",
-  "unique_contexts": 3388,
-  "total_transitions": 36898936
+  "unique_contexts": 3387,
+  "total_transitions": 36651893
 }

models/subword_markov/bar_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc48393a4abf97a7708eb8626bb77b238a51c44b6bba67b7e1d4fd0bd11ec189
-size 1029228
+oid sha256:721af06eca9bce8df729cc27b5e4d085cc14c4169ca0eaa6f9b413436eea4eaf
+size 1045675

models/subword_markov/bar_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "bar",
-  "unique_contexts": 22382,
-  "total_transitions": 36872004
+  "unique_contexts": 22370,
+  "total_transitions": 36625125
 }

models/subword_markov/bar_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2791996798e11cff4740288bb4e88ef150e9193843760570285c334a6ad21342
-size 4585569
+oid sha256:85e77908e5588d8b3e0a2130ebbb17e7e819804b700ceb76778e3903f0685824
+size 4586691

models/subword_markov/bar_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "bar",
-  "unique_contexts": 130801,
-  "total_transitions": 36845072
+  "unique_contexts": 130496,
+  "total_transitions": 36598357
 }

models/subword_markov/bar_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19edd5d2767e201a87efc206fec70368c6dc23b8ed658bbf39f26635b1f373d3
-size 15789022
+oid sha256:b9f51ef1a5af0b3612eee973a327c263d507f07902c3cd619a57ce26fd66a327
+size 15805911

models/subword_markov/bar_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "bar",
-  "unique_contexts": 610360,
-  "total_transitions": 36818140
+  "unique_contexts": 608299,
+  "total_transitions": 36571589
 }

models/subword_ngram/bar_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e38e7d14a00c23438ba07cef06a0be700a6eaea54c9aa917c3319ce7eba165f
-size 102154
+oid sha256:8c233f224c3b59afb31134f4f3a3a33b53e2254dab8f9f04674d0001616163fb
+size 101595

models/subword_ngram/bar_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "bar",
-  "unique_ngrams": 7805,
-  "total_ngrams": 36898936
+  "unique_ngrams": 7796,
+  "total_ngrams": 36651893
 }

models/subword_ngram/bar_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff09d3ef6e1cfa552449a8dedd31d9754c60b8a91db7126c3cc9c4c71957830a
-size 780954
+oid sha256:03ebf42dad05edb6845665a25f8715e78a80a513cc71353a9f9ed89896654aec
+size 785124

models/subword_ngram/bar_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "bar",
-  "unique_ngrams": 63080,
-  "total_ngrams": 36872004
+  "unique_ngrams": 62893,
+  "total_ngrams": 36625125
 }

models/subword_ngram/bar_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72f067181a08a87d97a0065d3a1bdaeb78404363053efae74d287fdc5e58187b
-size 4090739
+oid sha256:f88d8053bb1dc1eb65008ebe3ab864b46b1fe54991cdd80b3695fa61f040eef0
+size 4069838

models/subword_ngram/bar_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "bar",
-  "unique_ngrams": 363810,
-  "total_ngrams": 36845072
+  "unique_ngrams": 362482,
+  "total_ngrams": 36598357
 }

models/subword_ngram/bar_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:274cb447c4330f623de246bc0f11c662c8241a357318d29c6f56be180ea273cc
+size 13233464

models/subword_ngram/bar_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "bar",
+  "unique_ngrams": 1153187,
+  "total_ngrams": 36571589
+}

models/tokenizer/bar_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70ddb2da3b8a1d35b41491bf8a0ea06f6e0357f42e81f619258dafcbc3567d87
-size 500516
+oid sha256:2f0ad7f979639aaa3a4633a0d1d32b9d0b65285c5e3dae9f5828891b7b85abc3
+size 500310

models/tokenizer/bar_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7c6a1964c1d0a5e0f3f657f5fe39d0fc4524fef607b13be5a9d6be1c48e71b2
-size 773157
+oid sha256:59601d467e0da057bd05285352a21240b3dd48ee44ebe4fc932982ddd5c7781d
+size 773259

models/tokenizer/bar_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a6c179e6feecfed4b2ee3d6bb1547e591a0fdece5e1f9b05bc8dc198ae6ac1b
-size 1335748
+oid sha256:16a70b812f876c520eda78b3906c91ea9188e0b222d4aba9a98e76837fefb1b9
+size 1335778

models/tokenizer/bar_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77089651ccfa512d13316ef10f75bd2ec3ead99b2fce243a47c46f50b404db47
-size 368818
+oid sha256:5d0053358bd61bafa827c9c2e178c9ac1975691336ff4dccbb4b66fe579aa31b
+size 368919

models/vocabulary/bar_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7edcd6cf37858bad1ad7f0f51a2247a6bb728fc9c8b48ffb32bfd46cfe4ee3f
-size 3431123
+oid sha256:7c18fc1c0e05d2e88fb6900bad3bce4ada4d362898d269a140e9797481a48870
+size 3414560

models/vocabulary/bar_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "bar",
-  "vocabulary_size": 213465,
+  "vocabulary_size": 212365,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.0995109201675251,
+    "type_token_ratio": 0.09984296589928575,
     "coverage": {
-      "top_100": 0.3198175845558876,
-      "top_1000": 0.5158986689356716,
-      "top_5000": 0.6559721791165527,
-      "top_10000": 0.7190383696641477
+      "top_100": 0.31994929966653296,
+      "top_1000": 0.515994919433343,
+      "top_5000": 0.6560289595810551,
+      "top_10000": 0.7190537927378092
     },
-    "hapax_count": 357256,
-    "hapax_ratio": 0.6259731112049496,
-    "total_documents": 26932
+    "hapax_count": 356362,
+    "hapax_ratio": 0.6265958887128622,
+    "total_documents": 26768
   }
 }

models/word_markov/bar_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51608c66d7996b04b88bec1cf78dd596b044f1df0c12652fcb092ce187c00456
-size 29383606
+oid sha256:8eac799a7473b34a018b368f01554e07d06108fc7be5ce6633307e3764c1a01d
+size 29188879

models/word_markov/bar_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "bar",
-  "unique_contexts": 569846,
-  "total_transitions": 5708328
+  "unique_contexts": 567851,
+  "total_transitions": 5669447
 }

models/word_markov/bar_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fa0c4b83dfa569be9152f9aa90cd11f1e0d964b9f72a6e43fd8f8c02a1c4229
-size 62105262
+oid sha256:651b58594c9f64099a4dfebd6df479aecb9f4974a103f0f607946c935fa2562d
+size 61707647

models/word_markov/bar_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "bar",
-  "unique_contexts": 2948968,
-  "total_transitions": 5681396
+  "unique_contexts": 2930161,
+  "total_transitions": 5642679
 }