Pre-processing #3
Description
fastText did some pre-processing before training, so it's safest to do the same, especially lowercasing:
> (async () => { console.log(await lid.predict("Ótimo")) })()
> [ { lang: 'hu', prob: 0.3540060520172119 } ]
> (async () => { console.log(await lid.predict("ótimo")) })()
> [ { lang: 'pt', prob: 0.9964836835861206 } ]
> (async () => { console.log(await lid.predict("Uma boa experiencia")) })()
> [ { lang: 'en', prob: 0.5653783679008484 } ]
> (async () => { console.log(await lid.predict("uma boa experiencia")) })()
> [ { lang: 'pt', prob: 0.9949126839637756 } ]