|
|
|
|
@@ -12,100 +12,119 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
|
|
|
#[allow(non_camel_case_types)]
|
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
|
|
|
|
pub enum LlmModelId {
|
|
|
|
|
#[serde(rename = "qwen3_1_7b")]
|
|
|
|
|
Qwen3_1_7B_Q4,
|
|
|
|
|
#[serde(rename = "qwen3_4b_instruct_2507")]
|
|
|
|
|
Qwen3_4BInstruct2507Q4,
|
|
|
|
|
#[serde(rename = "qwen3_14b")]
|
|
|
|
|
Qwen3_14BQ5,
|
|
|
|
|
#[serde(rename = "qwen3_5_2b")]
|
|
|
|
|
Qwen3_5_2B_Q4,
|
|
|
|
|
#[serde(rename = "qwen3_5_4b")]
|
|
|
|
|
Qwen3_5_4B_Q4,
|
|
|
|
|
#[serde(rename = "qwen3_5_9b")]
|
|
|
|
|
Qwen3_5_9B_Q4,
|
|
|
|
|
#[serde(rename = "qwen3_6_27b")]
|
|
|
|
|
Qwen3_6_27B_Q4,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl LlmModelId {
|
|
|
|
|
pub fn default_tier() -> Self {
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4
|
|
|
|
|
Self::Qwen3_5_4B_Q4
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn as_str(&self) -> &'static str {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => "qwen3_1_7b",
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => "qwen3_4b_instruct_2507",
|
|
|
|
|
Self::Qwen3_14BQ5 => "qwen3_14b",
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => "qwen3_5_2b",
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => "qwen3_5_4b",
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => "qwen3_5_9b",
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => "qwen3_6_27b",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn display_name(&self) -> &'static str {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => "Qwen3 1.7B",
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => "Qwen3 4B Instruct 2507",
|
|
|
|
|
Self::Qwen3_14BQ5 => "Qwen3 14B",
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => "Qwen3.5 2B",
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => "Qwen3.5 4B",
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => "Qwen3.5 9B",
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => "Qwen3.6 27B",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn file_name(&self) -> &'static str {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => "Qwen3-1.7B-Q4_K_M.gguf",
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => "Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
|
|
|
|
|
Self::Qwen3_14BQ5 => "Qwen3-14B-Q5_K_M.gguf",
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => "Qwen3.5-2B-Q4_K_M.gguf",
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => "Qwen3.5-4B-Q4_K_M.gguf",
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => "Qwen3.5-9B-Q4_K_M.gguf",
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => "Qwen3.6-27B-Q4_K_M.gguf",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn size_bytes(&self) -> u64 {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => 1_107_409_472,
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => 2_497_281_120,
|
|
|
|
|
Self::Qwen3_14BQ5 => 10_514_570_624,
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => 1_280_835_840,
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => 2_740_937_888,
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => 5_680_522_464,
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => 16_817_244_384,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn minimum_ram_bytes(&self) -> u64 {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => 8 * 1024_u64.pow(3),
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => 16 * 1024_u64.pow(3),
|
|
|
|
|
Self::Qwen3_14BQ5 => 32 * 1024_u64.pow(3),
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => 8 * 1024_u64.pow(3),
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => 16 * 1024_u64.pow(3),
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => 32 * 1024_u64.pow(3),
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => 64 * 1024_u64.pow(3),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn recommended_vram_bytes(&self) -> Option<u64> {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => None,
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => Some(8 * 1024_u64.pow(3)),
|
|
|
|
|
Self::Qwen3_14BQ5 => Some(16 * 1024_u64.pow(3)),
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => None,
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => Some(6 * 1024_u64.pow(3)),
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => Some(12 * 1024_u64.pow(3)),
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => Some(24 * 1024_u64.pow(3)),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn description(&self) -> &'static str {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => "Low tier for 8 GB RAM and CPU-heavy machines.",
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => {
|
|
|
|
|
"Default tier for cleanup and task extraction on 16 GB systems."
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => "Minimal tier for 8 GB RAM and CPU-heavy machines.",
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => {
|
|
|
|
|
"Standard tier for cleanup and task extraction on 16 GB systems."
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => "High tier for 32 GB RAM with a 12 GB+ GPU.",
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => {
|
|
|
|
|
"Maximum tier for 64 GB RAM with a 24 GB GPU; partial CPU offload below that."
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_14BQ5 => "High tier for 32 GB+ RAM and larger GPUs.",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn hf_url(&self) -> &'static str {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3-1.7B-GGUF/resolve/d7f544eead698dbd1f15126ef60b45a1e1933222/Qwen3-1.7B-Q4_K_M.gguf"
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/f6d5376be1edb4d416d56da11e5397a961aca8ae/Qwen3.5-2B-Q4_K_M.gguf"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/a06e946bb6b655725eafa393f4a9745d460374c9/Qwen3-4B-Instruct-2507-Q4_K_M.gguf"
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/e87f176479d0855a907a41277aca2f8ee7a09523/Qwen3.5-4B-Q4_K_M.gguf"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_14BQ5 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3-14B-GGUF/resolve/a04a82c4739b3ef5fa6da7d10261db2c67dd1985/Qwen3-14B-Q5_K_M.gguf"
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/Qwen3.5-9B-Q4_K_M.gguf"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => {
|
|
|
|
|
"https://huggingface.co/unsloth/Qwen3.6-27B-GGUF/resolve/82d411acf4a06cfb8d9b073a5211bf410bfc29bf/Qwen3.6-27B-Q4_K_M.gguf"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn sha256(&self) -> &'static str {
|
|
|
|
|
match self {
|
|
|
|
|
Self::Qwen3_1_7B_Q4 => {
|
|
|
|
|
"de942b0819216caa3bfe487180dd1bb37398fa1c98cb42bb0bbac7ab7d6e8a12"
|
|
|
|
|
Self::Qwen3_5_2B_Q4 => {
|
|
|
|
|
"aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_4BInstruct2507Q4 => {
|
|
|
|
|
"bf52d44a54b81d44219833556849529ee96f09da673a38783dddc2e2eaf17881"
|
|
|
|
|
Self::Qwen3_5_4B_Q4 => {
|
|
|
|
|
"00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_5_9B_Q4 => {
|
|
|
|
|
"03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_6_27B_Q4 => {
|
|
|
|
|
"5ed60d0af4650a854b1755bd392f9aef4872643dc25a254bc68043fa638392a0"
|
|
|
|
|
}
|
|
|
|
|
Self::Qwen3_14BQ5 => "6f87abc471bd509ad46aca4284b3cfa926d8114bc491bb0a7a3a7f74c16ef95b",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -121,9 +140,10 @@ impl FromStr for LlmModelId {
|
|
|
|
|
|
|
|
|
|
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
|
|
|
|
match value {
|
|
|
|
|
"qwen3_1_7b" => Ok(Self::Qwen3_1_7B_Q4),
|
|
|
|
|
"qwen3_4b_instruct_2507" => Ok(Self::Qwen3_4BInstruct2507Q4),
|
|
|
|
|
"qwen3_14b" => Ok(Self::Qwen3_14BQ5),
|
|
|
|
|
"qwen3_5_2b" => Ok(Self::Qwen3_5_2B_Q4),
|
|
|
|
|
"qwen3_5_4b" => Ok(Self::Qwen3_5_4B_Q4),
|
|
|
|
|
"qwen3_5_9b" => Ok(Self::Qwen3_5_9B_Q4),
|
|
|
|
|
"qwen3_6_27b" => Ok(Self::Qwen3_6_27B_Q4),
|
|
|
|
|
other => Err(format!("Unknown LLM model id: {other}")),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -154,9 +174,10 @@ pub enum DownloadError {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const ALL_MODELS: &[LlmModelId] = &[
|
|
|
|
|
LlmModelId::Qwen3_1_7B_Q4,
|
|
|
|
|
LlmModelId::Qwen3_4BInstruct2507Q4,
|
|
|
|
|
LlmModelId::Qwen3_14BQ5,
|
|
|
|
|
LlmModelId::Qwen3_5_2B_Q4,
|
|
|
|
|
LlmModelId::Qwen3_5_4B_Q4,
|
|
|
|
|
LlmModelId::Qwen3_5_9B_Q4,
|
|
|
|
|
LlmModelId::Qwen3_6_27B_Q4,
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
static ACTIVE_DOWNLOADS: LazyLock<Mutex<std::collections::HashSet<LlmModelId>>> =
|
|
|
|
|
@@ -206,16 +227,15 @@ pub fn model_info(id: LlmModelId) -> LlmModelInfo {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn recommend_tier(total_ram_bytes: u64, total_vram_bytes: Option<u64>) -> LlmModelId {
|
|
|
|
|
if total_vram_bytes.unwrap_or(0) >= 16 * 1024_u64.pow(3)
|
|
|
|
|
&& total_ram_bytes >= 32 * 1024_u64.pow(3)
|
|
|
|
|
{
|
|
|
|
|
LlmModelId::Qwen3_14BQ5
|
|
|
|
|
} else if total_vram_bytes.unwrap_or(0) >= 8 * 1024_u64.pow(3)
|
|
|
|
|
|| total_ram_bytes >= 16 * 1024_u64.pow(3)
|
|
|
|
|
{
|
|
|
|
|
LlmModelId::Qwen3_4BInstruct2507Q4
|
|
|
|
|
let vram = total_vram_bytes.unwrap_or(0);
|
|
|
|
|
if vram >= 24 * 1024_u64.pow(3) && total_ram_bytes >= 64 * 1024_u64.pow(3) {
|
|
|
|
|
LlmModelId::Qwen3_6_27B_Q4
|
|
|
|
|
} else if vram >= 12 * 1024_u64.pow(3) && total_ram_bytes >= 32 * 1024_u64.pow(3) {
|
|
|
|
|
LlmModelId::Qwen3_5_9B_Q4
|
|
|
|
|
} else if vram >= 6 * 1024_u64.pow(3) || total_ram_bytes >= 16 * 1024_u64.pow(3) {
|
|
|
|
|
LlmModelId::Qwen3_5_4B_Q4
|
|
|
|
|
} else {
|
|
|
|
|
LlmModelId::Qwen3_1_7B_Q4
|
|
|
|
|
LlmModelId::Qwen3_5_2B_Q4
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -389,15 +409,15 @@ mod tests {
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn model_path_contains_model_dir_and_filename() {
|
|
|
|
|
let path = model_path(LlmModelId::Qwen3_1_7B_Q4);
|
|
|
|
|
assert!(path.to_string_lossy().ends_with("Qwen3-1.7B-Q4_K_M.gguf"));
|
|
|
|
|
let path = model_path(LlmModelId::Qwen3_5_2B_Q4);
|
|
|
|
|
assert!(path.to_string_lossy().ends_with("Qwen3.5-2B-Q4_K_M.gguf"));
|
|
|
|
|
assert!(path.starts_with(model_dir()));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn recommend_tier_prefers_mid_by_default() {
|
|
|
|
|
let tier = recommend_tier(16 * 1024_u64.pow(3), None);
|
|
|
|
|
assert_eq!(tier, LlmModelId::Qwen3_4BInstruct2507Q4);
|
|
|
|
|
assert_eq!(tier, LlmModelId::Qwen3_5_4B_Q4);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|