Database schema optimization - part 5

This commit is contained in:
mdecimus
2025-11-04 17:45:37 +01:00
parent 14bce0a346
commit a7815ffd6b
33 changed files with 1891 additions and 299 deletions

View File

@@ -11,6 +11,8 @@ pub mod stopwords;
use std::borrow::Cow;
use utils::config::utils::ParseValue;
use crate::tokenizers::{
Token, chinese::ChineseTokenizer, japanese::JapaneseTokenizer, word::WordTokenizer,
};
@@ -118,6 +120,10 @@ pub enum Language {
}
impl Language {
pub fn is_unknown(&self) -> bool {
matches!(self, Language::Unknown)
}
pub fn from_iso_639(code: &str) -> Option<Self> {
hashify::map!(
code.split_once('-').map(|c| c.0).unwrap_or(code).as_bytes(),
@@ -193,3 +199,9 @@ impl Language {
}
}
}
impl ParseValue for Language {
fn parse_value(value: &str) -> utils::config::Result<Self> {
Language::from_iso_639(value).ok_or_else(|| format!("Invalid language code: {}", value))
}
}