diff --git a/.gitignore b/.gitignore index a36b9f2..a8c2eee 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ Kokoro-82M/ ui/data/ EXTERNAL_UV_DOCUMENTATION* app +api/temp_files/ # Docker Dockerfile* diff --git a/api/src/core/config.py b/api/src/core/config.py index d22f427..44d4b8a 100644 --- a/api/src/core/config.py +++ b/api/src/core/config.py @@ -32,10 +32,12 @@ class Settings(BaseSettings): cors_origins: list[str] = ["*"] # CORS origins for web player cors_enabled: bool = True # Whether to enable CORS - # Temp File Settings + # Temp File Settings for WEB Ui temp_file_dir: str = "api/temp_files" # Directory for temporary audio files (relative to project root) max_temp_dir_size_mb: int = 2048 # Maximum size of temp directory (2GB) - temp_file_max_age_hours: int = 1 # Remove temp files older than 1 hour + max_temp_dir_age_hours: int = 1 # Remove temp files older than 1 hour + max_temp_dir_count: int = 3 # Maximum number of temp files to keep + class Config: env_file = ".env" diff --git a/api/src/core/paths.py b/api/src/core/paths.py index eeee704..0c6bfff 100644 --- a/api/src/core/paths.py +++ b/api/src/core/paths.py @@ -351,7 +351,7 @@ async def cleanup_temp_files() -> None: for entry in entries: if entry.is_file(): stat = await aiofiles.os.stat(entry.path) - max_age = stat.st_mtime + (settings.temp_file_max_age_hours * 3600) + max_age = stat.st_mtime + (settings.max_temp_dir_age_hours * 3600) if max_age < stat.st_mtime: try: await aiofiles.os.remove(entry.path) diff --git a/api/src/main.py b/api/src/main.py index 744977e..3c505d7 100644 --- a/api/src/main.py +++ b/api/src/main.py @@ -18,6 +18,7 @@ from .routers.web_player import router as web_router from .core.model_config import model_config from .routers.development import router as dev_router from .routers.openai_compatible import router as openai_router +from .routers.debug import router as debug_router from .services.tts_service import TTSService @@ -48,7 +49,7 @@ async def lifespan(app: FastAPI): """Lifespan context manager for model initialization""" from .inference.model_manager import get_manager from .inference.voice_manager import get_manager as get_voice_manager - from .core.paths import cleanup_temp_files + from .services.temp_manager import cleanup_temp_files # Clean old temp files on startup await cleanup_temp_files() @@ -130,6 +131,7 @@ if settings.cors_enabled: # Include routers app.include_router(openai_router, prefix="/v1") app.include_router(dev_router) # Development endpoints +app.include_router(debug_router) # Debug endpoints if settings.enable_web_player: app.include_router(web_router, prefix="/web") # Web player static files diff --git a/api/src/routers/debug.py b/api/src/routers/debug.py new file mode 100644 index 0000000..86fff94 --- /dev/null +++ b/api/src/routers/debug.py @@ -0,0 +1,134 @@ +from fastapi import APIRouter +import psutil +import threading +import time +from datetime import datetime +try: + import GPUtil + GPU_AVAILABLE = True +except ImportError: + GPU_AVAILABLE = False + +router = APIRouter(tags=["debug"]) + +@router.get("/debug/threads") +async def get_thread_info(): + process = psutil.Process() + current_threads = threading.enumerate() + + # Get per-thread CPU times + thread_details = [] + for thread in current_threads: + thread_info = { + "name": thread.name, + "id": thread.ident, + "alive": thread.is_alive(), + "daemon": thread.daemon + } + thread_details.append(thread_info) + + return { + "total_threads": process.num_threads(), + "active_threads": len(current_threads), + "thread_names": [t.name for t in current_threads], + "thread_details": thread_details, + "memory_mb": process.memory_info().rss / 1024 / 1024 + } + +@router.get("/debug/storage") +async def get_storage_info(): + # Get disk partitions + partitions = psutil.disk_partitions() + storage_info = [] + + for partition in partitions: + try: + usage = psutil.disk_usage(partition.mountpoint) + storage_info.append({ + "device": partition.device, + "mountpoint": partition.mountpoint, + "fstype": partition.fstype, + "total_gb": usage.total / (1024**3), + "used_gb": usage.used / (1024**3), + "free_gb": usage.free / (1024**3), + "percent_used": usage.percent + }) + except PermissionError: + continue + + return { + "storage_info": storage_info + } + +@router.get("/debug/system") +async def get_system_info(): + process = psutil.Process() + + # CPU Info + cpu_info = { + "cpu_count": psutil.cpu_count(), + "cpu_percent": psutil.cpu_percent(interval=1), + "per_cpu_percent": psutil.cpu_percent(interval=1, percpu=True), + "load_avg": psutil.getloadavg() + } + + # Memory Info + virtual_memory = psutil.virtual_memory() + swap_memory = psutil.swap_memory() + memory_info = { + "virtual": { + "total_gb": virtual_memory.total / (1024**3), + "available_gb": virtual_memory.available / (1024**3), + "used_gb": virtual_memory.used / (1024**3), + "percent": virtual_memory.percent + }, + "swap": { + "total_gb": swap_memory.total / (1024**3), + "used_gb": swap_memory.used / (1024**3), + "free_gb": swap_memory.free / (1024**3), + "percent": swap_memory.percent + } + } + + # Process Info + process_info = { + "pid": process.pid, + "status": process.status(), + "create_time": datetime.fromtimestamp(process.create_time()).isoformat(), + "cpu_percent": process.cpu_percent(), + "memory_percent": process.memory_percent(), + } + + # Network Info + network_info = { + "connections": len(process.net_connections()), + "network_io": psutil.net_io_counters()._asdict() + } + + # GPU Info if available + gpu_info = None + if GPU_AVAILABLE: + try: + gpus = GPUtil.getGPUs() + gpu_info = [{ + "id": gpu.id, + "name": gpu.name, + "load": gpu.load, + "memory": { + "total": gpu.memoryTotal, + "used": gpu.memoryUsed, + "free": gpu.memoryFree, + "percent": (gpu.memoryUsed / gpu.memoryTotal) * 100 + }, + "temperature": gpu.temperature + } for gpu in gpus] + except Exception: + gpu_info = "GPU information unavailable" + + return { + "cpu": cpu_info, + "memory": memory_info, + "process": process_info, + "network": network_info, + "gpu": gpu_info + } \ No newline at end of file diff --git a/api/src/services/temp_manager.py b/api/src/services/temp_manager.py index 63b4be5..bfb825c 100644 --- a/api/src/services/temp_manager.py +++ b/api/src/services/temp_manager.py @@ -2,14 +2,71 @@ import os import tempfile -from typing import Optional +from typing import Optional, List import aiofiles from fastapi import HTTPException from loguru import logger from ..core.config import settings -from ..core.paths import _scan_directories + + +async def cleanup_temp_files() -> None: + """Clean up old temp files""" + try: + if not await aiofiles.os.path.exists(settings.temp_file_dir): + await aiofiles.os.makedirs(settings.temp_file_dir, exist_ok=True) + return + + # Get all temp files with stats + files = [] + total_size = 0 + + # Use os.scandir for sync iteration, but aiofiles.os.stat for async stats + for entry in os.scandir(settings.temp_file_dir): + if entry.is_file(): + stat = await aiofiles.os.stat(entry.path) + files.append((entry.path, stat.st_mtime, stat.st_size)) + total_size += stat.st_size + + # Sort by modification time (oldest first) + files.sort(key=lambda x: x[1]) + + # Remove files if: + # 1. They're too old + # 2. We have too many files + # 3. Directory is too large + current_time = (await aiofiles.os.stat(settings.temp_file_dir)).st_mtime + max_age = settings.max_temp_dir_age_hours * 3600 + + for path, mtime, size in files: + should_delete = False + + # Check age + if current_time - mtime > max_age: + should_delete = True + logger.info(f"Deleting old temp file: {path}") + + # Check count limit + elif len(files) > settings.max_temp_dir_count: + should_delete = True + logger.info(f"Deleting excess temp file: {path}") + + # Check size limit + elif total_size > settings.max_temp_dir_size_mb * 1024 * 1024: + should_delete = True + logger.info(f"Deleting to reduce directory size: {path}") + + if should_delete: + try: + await aiofiles.os.remove(path) + total_size -= size + logger.info(f"Deleted temp file: {path}") + except Exception as e: + logger.warning(f"Failed to delete temp file {path}: {e}") + + except Exception as e: + logger.warning(f"Error during temp file cleanup: {e}") class TempFileWriter: @@ -27,21 +84,11 @@ class TempFileWriter: async def __aenter__(self): """Async context manager entry""" - # Check temp dir size by scanning - total_size = 0 - entries = await _scan_directories([settings.temp_file_dir]) - for entry in entries: - stat = await aiofiles.os.stat(os.path.join(settings.temp_file_dir, entry)) - total_size += stat.st_size - - if total_size >= settings.max_temp_dir_size_mb * 1024 * 1024: - raise HTTPException( - status_code=507, - detail="Temporary storage full. Please try again later." - ) + # Clean up old files first + await cleanup_temp_files() # Create temp file with proper extension - os.makedirs(settings.temp_file_dir, exist_ok=True) + await aiofiles.os.makedirs(settings.temp_file_dir, exist_ok=True) temp = tempfile.NamedTemporaryFile( dir=settings.temp_file_dir, delete=False, diff --git a/api/src/services/text_processing/chunker.py b/api/src/services/text_processing/chunker.py deleted file mode 100644 index 80e67f0..0000000 --- a/api/src/services/text_processing/chunker.py +++ /dev/null @@ -1,31 +0,0 @@ -# """Text chunking module for TTS processing""" - -# from typing import List, AsyncGenerator - -# async def fallback_split(text: str, max_chars: int = 400) -> List[str]: -# """Emergency length control - only used if chunks are too long""" -# words = text.split() -# chunks = [] -# current = [] -# current_len = 0 - -# for word in words: -# # Always include at least one word per chunk -# if not current: -# current.append(word) -# current_len = len(word) -# continue - -# # Check if adding word would exceed limit -# if current_len + len(word) + 1 <= max_chars: -# current.append(word) -# current_len += len(word) + 1 -# else: -# chunks.append(" ".join(current)) -# current = [word] -# current_len = len(word) - -# if current: -# chunks.append(" ".join(current)) - -# return chunks diff --git a/api/src/services/text_processing/text_processor.py b/api/src/services/text_processing/text_processor.py index 98fc45a..74f0f91 100644 --- a/api/src/services/text_processing/text_processor.py +++ b/api/src/services/text_processing/text_processor.py @@ -50,7 +50,7 @@ def process_text_chunk(text: str, language: str = "a") -> List[int]: async def yield_chunk(text: str, tokens: List[int], chunk_count: int) -> Tuple[str, List[int]]: """Yield a chunk with consistent logging.""" - logger.info(f"Yielding chunk {chunk_count}: '{text[:50]}...' ({len(tokens)} tokens)") + logger.debug(f"Yielding chunk {chunk_count}: '{text[:50]}...' ({len(tokens)} tokens)") return text, tokens def process_text(text: str, language: str = "a") -> List[int]: @@ -111,7 +111,7 @@ async def smart_split(text: str, max_tokens: int = ABSOLUTE_MAX) -> AsyncGenerat if current_chunk: chunk_text = " ".join(current_chunk) chunk_count += 1 - logger.info(f"Yielding chunk {chunk_count}: '{chunk_text[:50]}...' ({current_count} tokens)") + logger.debug(f"Yielding chunk {chunk_count}: '{chunk_text[:50]}...' ({current_count} tokens)") yield chunk_text, current_tokens current_chunk = [] current_tokens = [] @@ -144,7 +144,7 @@ async def smart_split(text: str, max_tokens: int = ABSOLUTE_MAX) -> AsyncGenerat if clause_chunk: chunk_text = " ".join(clause_chunk) chunk_count += 1 - logger.info(f"Yielding clause chunk {chunk_count}: '{chunk_text[:50]}...' ({clause_count} tokens)") + logger.debug(f"Yielding clause chunk {chunk_count}: '{chunk_text[:50]}...' ({clause_count} tokens)") yield chunk_text, clause_tokens clause_chunk = [full_clause] clause_tokens = tokens @@ -154,7 +154,7 @@ async def smart_split(text: str, max_tokens: int = ABSOLUTE_MAX) -> AsyncGenerat if clause_chunk: chunk_text = " ".join(clause_chunk) chunk_count += 1 - logger.info(f"Yielding final clause chunk {chunk_count}: '{chunk_text[:50]}...' ({clause_count} tokens)") + logger.debug(f"Yielding final clause chunk {chunk_count}: '{chunk_text[:50]}...' ({clause_count} tokens)") yield chunk_text, clause_tokens # Regular sentence handling diff --git a/api/tests/debug.http b/api/tests/debug.http new file mode 100644 index 0000000..ed76003 --- /dev/null +++ b/api/tests/debug.http @@ -0,0 +1,11 @@ +### Get Thread Information +GET http://localhost:8880/debug/threads +Accept: application/json + +### Get Storage Information +GET http://localhost:8880/debug/storage +Accept: application/json + +### Get System Information +GET http://localhost:8880/debug/system +Accept: application/json \ No newline at end of file diff --git a/api/tests/test_chunker.py b/api/tests/test_chunker.py deleted file mode 100644 index 002da72..0000000 --- a/api/tests/test_chunker.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for text chunking service""" - -from unittest.mock import patch - -import pytest - -from api.src.services.text_processing import chunker - - -@pytest.fixture(autouse=True) -def mock_settings(): - """Mock settings for all tests""" - with patch("api.src.services.text_processing.chunker.settings") as mock_settings: - mock_settings.max_chunk_size = 300 - yield mock_settings - - -def test_split_text(): - """Test text splitting into sentences""" - text = "First sentence. Second sentence! Third sentence?" - sentences = list(chunker.split_text(text)) - assert len(sentences) == 3 - assert sentences[0] == "First sentence." - assert sentences[1] == "Second sentence!" - assert sentences[2] == "Third sentence?" - - -def test_split_text_empty(): - """Test splitting empty text""" - assert list(chunker.split_text("")) == [] - - -def test_split_text_single_sentence(): - """Test splitting single sentence""" - text = "Just one sentence." - assert list(chunker.split_text(text)) == ["Just one sentence."] - - -def test_split_text_with_custom_chunk_size(): - """Test splitting with custom max chunk size""" - text = "First part, second part, third part." - chunks = list(chunker.split_text(text, max_chunk=15)) - assert len(chunks) == 3 - assert chunks[0] == "First part," - assert chunks[1] == "second part," - assert chunks[2] == "third part." diff --git a/pyproject.toml b/pyproject.toml index 6100bb0..a07d1a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "matplotlib>=3.10.0", "semchunk>=3.0.1", "mutagen>=1.47.0", + "psutil>=6.1.1", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index 94d313e..079f7d7 100644 --- a/uv.lock +++ b/uv.lock @@ -1015,6 +1015,7 @@ dependencies = [ { name = "numpy" }, { name = "openai" }, { name = "phonemizer" }, + { name = "psutil" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pydub" }, @@ -1068,6 +1069,7 @@ requires-dist = [ { name = "openai", specifier = ">=1.59.6" }, { name = "openai", marker = "extra == 'test'", specifier = ">=1.59.6" }, { name = "phonemizer", specifier = "==3.3.0" }, + { name = "psutil", specifier = ">=6.1.1" }, { name = "pydantic", specifier = "==2.10.4" }, { name = "pydantic-settings", specifier = "==2.7.0" }, { name = "pydub", specifier = ">=0.25.1" }, @@ -2111,6 +2113,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/b2/ab07b09e0f6d143dfb839693aa05765257bceaa13d03bf1a696b78323e7a/protobuf-5.29.3-py3-none-any.whl", hash = "sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f", size = 172550 }, ] +[[package]] +name = "psutil" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/5a/07871137bb752428aa4b659f910b399ba6f291156bdea939be3e96cae7cb/psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5", size = 508502 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/99/ca79d302be46f7bdd8321089762dd4476ee725fce16fc2b2e1dbba8cac17/psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8", size = 247511 }, + { url = "https://files.pythonhosted.org/packages/0b/6b/73dbde0dd38f3782905d4587049b9be64d76671042fdcaf60e2430c6796d/psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377", size = 248985 }, + { url = "https://files.pythonhosted.org/packages/17/38/c319d31a1d3f88c5b79c68b3116c129e5133f1822157dd6da34043e32ed6/psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003", size = 284488 }, + { url = "https://files.pythonhosted.org/packages/9c/39/0f88a830a1c8a3aba27fededc642da37613c57cbff143412e3536f89784f/psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160", size = 287477 }, + { url = "https://files.pythonhosted.org/packages/47/da/99f4345d4ddf2845cb5b5bd0d93d554e84542d116934fde07a0c50bd4e9f/psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3", size = 289017 }, + { url = "https://files.pythonhosted.org/packages/38/53/bd755c2896f4461fd4f36fa6a6dcb66a88a9e4b9fd4e5b66a77cf9d4a584/psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53", size = 250602 }, + { url = "https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649", size = 254444 }, +] + [[package]] name = "pycparser" version = "2.22" diff --git a/web/index.html b/web/index.html index 2e0ce73..a0bace6 100644 --- a/web/index.html +++ b/web/index.html @@ -84,6 +84,9 @@
+