import asyncio import pytest from llmux.vram_manager import VRAMManager, ModelSlot class FakeBackend: """Simulates a backend that tracks load/unload calls.""" def __init__(self): self.loaded = {} self.load_count = 0 self.unload_count = 0 async def load(self, model_id: str): self.loaded[model_id] = True self.load_count += 1 async def unload(self, model_id: str): self.loaded.pop(model_id, None) self.unload_count += 1 @pytest.fixture def manager(): return VRAMManager(total_vram_gb=16.0, verify_gpu=False) def test_priority_ordering(): assert ModelSlot.priority_rank("llm") == 0 assert ModelSlot.priority_rank("tts") == 1 assert ModelSlot.priority_rank("asr") == 2 @pytest.mark.asyncio async def test_load_into_empty_vram(manager): backend = FakeBackend() await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) assert manager.is_loaded("qwen3.5-4b") assert manager.available_vram_gb == pytest.approx(12.0) @pytest.mark.asyncio async def test_load_alongside_when_fits(manager): backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) assert manager.is_loaded("cohere-transcribe") assert manager.is_loaded("qwen3.5-4b") assert manager.available_vram_gb == pytest.approx(8.0) @pytest.mark.asyncio async def test_evict_llm_first(manager): backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("chatterbox-multilingual", model_type="tts", vram_gb=2.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) # 10 GB used. Loading 9B (9GB). Evict LLM (4B), free=12. ASR+TTS+9B=15, fits. await manager.load_model("qwen3.5-9b-fp8", model_type="llm", vram_gb=9.0, backend=backend) assert not manager.is_loaded("qwen3.5-4b") assert manager.is_loaded("cohere-transcribe") assert manager.is_loaded("chatterbox-multilingual") assert manager.is_loaded("qwen3.5-9b-fp8") @pytest.mark.asyncio async def test_evict_cascade_asr_survives(manager): """When LLM fits alongside ASR after evicting LLM+TTS, ASR survives.""" backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("chatterbox-multilingual", model_type="tts", vram_gb=2.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) # 10 GB used. Need 12GB. Evict LLM(4)->free=10. Evict TTS(2)->free=12. ASR+12=16, fits. await manager.load_model("large-llm", model_type="llm", vram_gb=12.0, backend=backend) assert not manager.is_loaded("qwen3.5-4b") assert not manager.is_loaded("chatterbox-multilingual") assert manager.is_loaded("cohere-transcribe") # ASR survives assert manager.is_loaded("large-llm") @pytest.mark.asyncio async def test_evict_cascade_full_for_huge_llm(manager): """When LLM is too large to fit alongside ASR, everything gets evicted.""" backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("chatterbox-multilingual", model_type="tts", vram_gb=2.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) # 10 GB used. gpt-oss-20b needs 13GB. Evict LLM(4)->free=10. TTS(2)->free=12. ASR(4)->free=16. Load alone. await manager.load_model("gpt-oss-20b", model_type="llm", vram_gb=13.0, backend=backend) assert not manager.is_loaded("qwen3.5-4b") assert not manager.is_loaded("chatterbox-multilingual") assert not manager.is_loaded("cohere-transcribe") # ASR evicted as last resort assert manager.is_loaded("gpt-oss-20b") @pytest.mark.asyncio async def test_tts_cannot_evict_asr(manager): """TTS request must not evict ASR — it evicts LLM instead.""" backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("qwen3.5-9b-fp8", model_type="llm", vram_gb=9.0, backend=backend) # 13GB used, 3GB free. TTS needs 2GB — fits! Load alongside. await manager.load_model("chatterbox", model_type="tts", vram_gb=2.0, backend=backend) assert manager.is_loaded("cohere-transcribe") assert manager.is_loaded("qwen3.5-9b-fp8") assert manager.is_loaded("chatterbox") # Now replace TTS with a bigger one that needs eviction # 15GB used, 1GB free. New TTS needs 2GB. Evict old TTS(2)->free=3. Load. await manager.load_model("chatterbox-ml", model_type="tts", vram_gb=2.0, backend=backend) assert manager.is_loaded("cohere-transcribe") # ASR must survive assert manager.is_loaded("chatterbox-ml") @pytest.mark.asyncio async def test_asr_evicts_llm_not_reversed(manager): """When ASR request arrives and LLM is loaded, evict LLM (lower priority).""" backend = FakeBackend() await manager.load_model("gpt-oss-20b", model_type="llm", vram_gb=13.0, backend=backend) # 13GB used, 3GB free. ASR needs 4GB. Must evict LLM. await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) assert not manager.is_loaded("gpt-oss-20b") assert manager.is_loaded("cohere-transcribe") @pytest.mark.asyncio async def test_already_loaded_is_noop(manager): backend = FakeBackend() await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) assert backend.load_count == 1 @pytest.mark.asyncio async def test_spec_scenario_switch_to_9b(manager): backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("chatterbox-multilingual", model_type="tts", vram_gb=2.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) await manager.load_model("qwen3.5-9b-fp8", model_type="llm", vram_gb=9.0, backend=backend) assert manager.is_loaded("cohere-transcribe") assert manager.is_loaded("chatterbox-multilingual") assert manager.is_loaded("qwen3.5-9b-fp8") assert not manager.is_loaded("qwen3.5-4b") assert manager.available_vram_gb == pytest.approx(1.0) @pytest.mark.asyncio async def test_get_loaded_models(manager): backend = FakeBackend() await manager.load_model("cohere-transcribe", model_type="asr", vram_gb=4.0, backend=backend) await manager.load_model("qwen3.5-4b", model_type="llm", vram_gb=4.0, backend=backend) loaded = manager.get_loaded_models() assert set(loaded.keys()) == {"cohere-transcribe", "qwen3.5-4b"}