diff --git a/.coveragerc b/.coveragerc index dab8655..c66579a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,6 +6,7 @@ omit = Kokoro-82M/* MagicMock/* test_*.py + examples/* [report] exclude_lines = diff --git a/CHANGELOG.md b/CHANGELOG.md index 4194878..6303af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ Notable changes to this project will be documented in this file. +## 2025-01-04 +### Added +- ONNX Support: + - Added single batch ONNX support for CPU inference + - Roughly 0.4 RTF (2.4x real-time speed) + +### Modified +- Code Refactoring: + - Work on modularizing phonemizer and tokenizer into separate services + - Incorporated these services into a dev endpoint +- Testing and Benchmarking: + - Cleaned up benchmarking scripts + - Cleaned up test scripts + - Added auto-WAV validation scripts ## 2025-01-02 - Audio Format Support: diff --git a/README.md b/README.md index da78651..2e38852 100644 --- a/README.md +++ b/README.md @@ -187,15 +187,13 @@ Key Performance Metrics: GPU Vs. CPU ```bash -# GPU: Requires NVIDIA GPU with CUDA 12.1 support +# GPU: Requires NVIDIA GPU with CUDA 12.1 support (~35x realtime speed) docker compose up --build -# CPU: ~10x slower than GPU inference +# CPU: ONNX optimized inference (~2.4x realtime speed) docker compose -f docker-compose.cpu.yml up --build ``` -*Note: CPU Inference is currently a very basic implementation, and not heavily tested* -
diff --git a/docker-compose.cpu.yml b/docker-compose.cpu.yml index d5e1a85..f44f2d4 100644 --- a/docker-compose.cpu.yml +++ b/docker-compose.cpu.yml @@ -47,14 +47,14 @@ services: model-fetcher: condition: service_healthy - # # Gradio UI service [Comment out everything below if you don't need it] - # gradio-ui: - # build: - # context: ./ui - # ports: - # - "7860:7860" - # volumes: - # - ./ui/data:/app/ui/data - # - ./ui/app.py:/app/app.py # Mount app.py for hot reload - # environment: - # - GRADIO_WATCH=True # Enable hot reloading + # Gradio UI service [Comment out everything below if you don't need it] + gradio-ui: + build: + context: ./ui + ports: + - "7860:7860" + volumes: + - ./ui/data:/app/ui/data + - ./ui/app.py:/app/app.py # Mount app.py for hot reload + environment: + - GRADIO_WATCH=True # Enable hot reloading diff --git a/docker-compose.yml b/docker-compose.yml index 7308745..2e7a86f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,14 +46,14 @@ services: model-fetcher: condition: service_healthy - # # Gradio UI service [Comment out everything below if you don't need it] - # gradio-ui: - # build: - # context: ./ui - # ports: - # - "7860:7860" - # volumes: - # - ./ui/data:/app/ui/data - # - ./ui/app.py:/app/app.py # Mount app.py for hot reload - # environment: - # - GRADIO_WATCH=True # Enable hot reloading + # Gradio UI service [Comment out everything below if you don't need it] + gradio-ui: + build: + context: ./ui + ports: + - "7860:7860" + volumes: + - ./ui/data:/app/ui/data + - ./ui/app.py:/app/app.py # Mount app.py for hot reload + environment: + - GRADIO_WATCH=True # Enable hot reloading diff --git a/examples/assorted_checks/benchmarks/benchmark_tts_rtf.py b/examples/assorted_checks/benchmarks/benchmark_tts_rtf.py index c55a136..385abb0 100644 --- a/examples/assorted_checks/benchmarks/benchmark_tts_rtf.py +++ b/examples/assorted_checks/benchmarks/benchmark_tts_rtf.py @@ -60,11 +60,11 @@ def main(): # Initialize system monitor monitor = SystemMonitor(interval=1.0) # 1 second interval # Set prefix for output files (e.g. "gpu", "cpu", "onnx", etc.) - prefix = "cpu_2_1_seq" + prefix = "gpu" # Generate token sizes if 'gpu' in prefix: token_sizes = generate_token_sizes( - max_tokens=3000, dense_step=150, + max_tokens=5000, dense_step=150, dense_max=1000, sparse_step=1000) elif 'cpu' in prefix: token_sizes = generate_token_sizes( diff --git a/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_results_rtf.json b/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_results_rtf.json index ae59bfe..ccac37e 100644 --- a/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_results_rtf.json +++ b/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_results_rtf.json @@ -2,606 +2,1252 @@ "results": [ { "tokens": 150, - "processing_time": 1.03, + "processing_time": 1.86, "output_length": 45.9, - "rtf": 0.02, - "elapsed_time": 1.07 + "rtf": 0.04, + "elapsed_time": 1.92 }, { "tokens": 300, - "processing_time": 2.51, + "processing_time": 3.08, "output_length": 96.425, "rtf": 0.03, - "elapsed_time": 3.63 + "elapsed_time": 5.06 }, { "tokens": 450, - "processing_time": 3.69, + "processing_time": 4.4, "output_length": 143.1, "rtf": 0.03, - "elapsed_time": 7.37 + "elapsed_time": 9.53 }, { "tokens": 600, - "processing_time": 5.52, + "processing_time": 6.47, "output_length": 188.675, "rtf": 0.03, - "elapsed_time": 12.96 + "elapsed_time": 16.06 }, { "tokens": 750, - "processing_time": 6.32, + "processing_time": 8.32, "output_length": 236.7, - "rtf": 0.03, - "elapsed_time": 19.34 + "rtf": 0.04, + "elapsed_time": 24.45 }, { "tokens": 900, - "processing_time": 8.4, + "processing_time": 8.92, "output_length": 283.425, "rtf": 0.03, - "elapsed_time": 27.82 + "elapsed_time": 33.45 }, { "tokens": 2000, - "processing_time": 15.46, + "processing_time": 18.55, "output_length": 624.325, - "rtf": 0.02, - "elapsed_time": 43.4 + "rtf": 0.03, + "elapsed_time": 52.14 }, { "tokens": 3000, - "processing_time": 27.11, + "processing_time": 23.98, "output_length": 931.15, "rtf": 0.03, - "elapsed_time": 70.7 + "elapsed_time": 76.32 + }, + { + "tokens": 4000, + "processing_time": 32.93, + "output_length": 1222.1, + "rtf": 0.03, + "elapsed_time": 109.53 + }, + { + "tokens": 5000, + "processing_time": 45.39, + "output_length": 1524.575, + "rtf": 0.03, + "elapsed_time": 155.23 } ], "system_metrics": [ { - "timestamp": "2025-01-03T17:45:10.797646", - "cpu_percent": 10.05, - "ram_percent": 54.2, - "ram_used_gb": 34.474674224853516, - "gpu_memory_used": 3992.0, - "relative_time": 0.06637930870056152 + "timestamp": "2025-01-04T02:37:52.172368", + "cpu_percent": 11.51, + "ram_percent": 52.8, + "ram_used_gb": 33.61172866821289, + "gpu_memory_used": 3216.0, + "relative_time": 0.08031892776489258 }, { - "timestamp": "2025-01-03T17:45:11.871315", + "timestamp": "2025-01-04T02:37:53.266071", + "cpu_percent": 15.33, + "ram_percent": 52.9, + "ram_used_gb": 33.678314208984375, + "gpu_memory_used": 3392.0, + "relative_time": 1.1673684120178223 + }, + { + "timestamp": "2025-01-04T02:37:54.352909", + "cpu_percent": 15.3, + "ram_percent": 53.0, + "ram_used_gb": 33.712764739990234, + "gpu_memory_used": 3667.0, + "relative_time": 2.253591537475586 + }, + { + "timestamp": "2025-01-04T02:37:55.439413", + "cpu_percent": 23.51, + "ram_percent": 52.7, + "ram_used_gb": 33.49789810180664, + "gpu_memory_used": 3662.0, + "relative_time": 3.3292760848999023 + }, + { + "timestamp": "2025-01-04T02:37:56.511211", + "cpu_percent": 14.69, + "ram_percent": 52.7, + "ram_used_gb": 33.494102478027344, + "gpu_memory_used": 3668.0, + "relative_time": 4.397106885910034 + }, + { + "timestamp": "2025-01-04T02:37:57.582176", + "cpu_percent": 11.01, + "ram_percent": 52.8, + "ram_used_gb": 33.564491271972656, + "gpu_memory_used": 3665.0, + "relative_time": 5.46670126914978 + }, + { + "timestamp": "2025-01-04T02:37:58.637969", + "cpu_percent": 15.04, + "ram_percent": 52.8, + "ram_used_gb": 33.555362701416016, + "gpu_memory_used": 3668.0, + "relative_time": 6.523184061050415 + }, + { + "timestamp": "2025-01-04T02:37:59.700880", + "cpu_percent": 13.32, + "ram_percent": 52.8, + "ram_used_gb": 33.559967041015625, + "gpu_memory_used": 3668.0, + "relative_time": 7.589032888412476 + }, + { + "timestamp": "2025-01-04T02:38:00.773895", + "cpu_percent": 12.45, + "ram_percent": 52.8, + "ram_used_gb": 33.609134674072266, + "gpu_memory_used": 3667.0, + "relative_time": 8.677486181259155 + }, + { + "timestamp": "2025-01-04T02:38:01.851195", + "cpu_percent": 12.62, + "ram_percent": 52.9, + "ram_used_gb": 33.67635726928711, + "gpu_memory_used": 3665.0, + "relative_time": 9.734971046447754 + }, + { + "timestamp": "2025-01-04T02:38:02.907897", + "cpu_percent": 20.61, + "ram_percent": 53.0, + "ram_used_gb": 33.72555160522461, + "gpu_memory_used": 3660.0, + "relative_time": 10.813292026519775 + }, + { + "timestamp": "2025-01-04T02:38:03.996322", + "cpu_percent": 33.24, + "ram_percent": 53.2, + "ram_used_gb": 33.832088470458984, + "gpu_memory_used": 3660.0, + "relative_time": 11.917856454849243 + }, + { + "timestamp": "2025-01-04T02:38:05.101973", + "cpu_percent": 14.24, + "ram_percent": 53.0, + "ram_used_gb": 33.7408447265625, + "gpu_memory_used": 3662.0, + "relative_time": 12.986546277999878 + }, + { + "timestamp": "2025-01-04T02:38:06.162037", + "cpu_percent": 14.38, + "ram_percent": 53.1, + "ram_used_gb": 33.774169921875, + "gpu_memory_used": 3662.0, + "relative_time": 14.062608242034912 + }, + { + "timestamp": "2025-01-04T02:38:07.248210", + "cpu_percent": 14.39, + "ram_percent": 53.2, + "ram_used_gb": 33.83738327026367, + "gpu_memory_used": 4029.0, + "relative_time": 15.156044960021973 + }, + { + "timestamp": "2025-01-04T02:38:08.329582", + "cpu_percent": 31.18, + "ram_percent": 53.2, + "ram_used_gb": 33.87126541137695, + "gpu_memory_used": 4032.0, + "relative_time": 16.249940395355225 + }, + { + "timestamp": "2025-01-04T02:38:09.432992", + "cpu_percent": 19.33, + "ram_percent": 53.2, + "ram_used_gb": 33.842403411865234, + "gpu_memory_used": 4032.0, + "relative_time": 17.331223011016846 + }, + { + "timestamp": "2025-01-04T02:38:10.505101", + "cpu_percent": 13.34, + "ram_percent": 53.2, + "ram_used_gb": 33.86738967895508, + "gpu_memory_used": 4029.0, + "relative_time": 18.390397548675537 + }, + { + "timestamp": "2025-01-04T02:38:11.570033", + "cpu_percent": 12.61, + "ram_percent": 53.4, + "ram_used_gb": 33.938289642333984, + "gpu_memory_used": 4028.0, + "relative_time": 19.477521181106567 + }, + { + "timestamp": "2025-01-04T02:38:12.663780", + "cpu_percent": 15.78, + "ram_percent": 53.4, + "ram_used_gb": 33.969398498535156, + "gpu_memory_used": 4030.0, + "relative_time": 20.57425808906555 + }, + { + "timestamp": "2025-01-04T02:38:13.750065", + "cpu_percent": 18.69, + "ram_percent": 53.5, + "ram_used_gb": 34.03954315185547, + "gpu_memory_used": 4021.0, + "relative_time": 21.652076244354248 + }, + { + "timestamp": "2025-01-04T02:38:14.825318", + "cpu_percent": 10.48, + "ram_percent": 53.6, + "ram_used_gb": 34.07048416137695, + "gpu_memory_used": 4025.0, + "relative_time": 22.73010230064392 + }, + { + "timestamp": "2025-01-04T02:38:15.912340", + "cpu_percent": 12.53, + "ram_percent": 53.6, + "ram_used_gb": 34.09389877319336, + "gpu_memory_used": 4026.0, + "relative_time": 23.81609869003296 + }, + { + "timestamp": "2025-01-04T02:38:17.003329", + "cpu_percent": 16.09, + "ram_percent": 53.7, + "ram_used_gb": 34.1781120300293, + "gpu_memory_used": 4025.0, + "relative_time": 24.90904140472412 + }, + { + "timestamp": "2025-01-04T02:38:18.079837", + "cpu_percent": 14.98, + "ram_percent": 53.8, + "ram_used_gb": 34.21260070800781, + "gpu_memory_used": 4025.0, + "relative_time": 25.986279249191284 + }, + { + "timestamp": "2025-01-04T02:38:19.167635", + "cpu_percent": 14.85, + "ram_percent": 53.8, + "ram_used_gb": 34.23923873901367, + "gpu_memory_used": 4024.0, + "relative_time": 27.076823234558105 + }, + { + "timestamp": "2025-01-04T02:38:20.258141", + "cpu_percent": 15.05, + "ram_percent": 53.9, + "ram_used_gb": 34.26483917236328, + "gpu_memory_used": 4015.0, + "relative_time": 28.144607067108154 + }, + { + "timestamp": "2025-01-04T02:38:21.315694", + "cpu_percent": 17.08, + "ram_percent": 53.9, + "ram_used_gb": 34.31473922729492, + "gpu_memory_used": 4016.0, + "relative_time": 29.20189356803894 + }, + { + "timestamp": "2025-01-04T02:38:22.388259", + "cpu_percent": 17.47, + "ram_percent": 54.0, + "ram_used_gb": 34.35490798950195, + "gpu_memory_used": 4016.0, + "relative_time": 30.28918957710266 + }, + { + "timestamp": "2025-01-04T02:38:23.463469", + "cpu_percent": 15.76, + "ram_percent": 54.0, + "ram_used_gb": 34.33717346191406, + "gpu_memory_used": 4002.0, + "relative_time": 31.364880561828613 + }, + { + "timestamp": "2025-01-04T02:38:24.540334", "cpu_percent": 13.54, - "ram_percent": 54.2, - "ram_used_gb": 34.47991180419922, - "gpu_memory_used": 3990.0, - "relative_time": 1.1326591968536377 + "ram_percent": 54.1, + "ram_used_gb": 34.38197708129883, + "gpu_memory_used": 3999.0, + "relative_time": 32.4253191947937 }, { - "timestamp": "2025-01-03T17:45:12.932597", - "cpu_percent": 12.76, + "timestamp": "2025-01-04T02:38:25.597934", + "cpu_percent": 13.99, "ram_percent": 54.2, - "ram_used_gb": 34.501747131347656, - "gpu_memory_used": 3990.0, - "relative_time": 2.192795515060425 + "ram_used_gb": 34.48365783691406, + "gpu_memory_used": 4004.0, + "relative_time": 33.50029754638672 }, { - "timestamp": "2025-01-03T17:45:13.995052", - "cpu_percent": 15.48, + "timestamp": "2025-01-04T02:38:26.673108", + "cpu_percent": 15.16, "ram_percent": 54.2, - "ram_used_gb": 34.48517990112305, - "gpu_memory_used": 3989.0, - "relative_time": 3.253366231918335 + "ram_used_gb": 34.50083923339844, + "gpu_memory_used": 4011.0, + "relative_time": 34.5756139755249 }, { - "timestamp": "2025-01-03T17:45:15.056310", - "cpu_percent": 11.96, + "timestamp": "2025-01-04T02:38:27.748147", + "cpu_percent": 17.68, "ram_percent": 54.2, - "ram_used_gb": 34.457679748535156, - "gpu_memory_used": 3980.0, - "relative_time": 4.331450462341309 + "ram_used_gb": 34.49884033203125, + "gpu_memory_used": 4016.0, + "relative_time": 35.650988817214966 }, { - "timestamp": "2025-01-03T17:45:16.128795", - "cpu_percent": 14.28, - "ram_percent": 54.2, - "ram_used_gb": 34.465850830078125, - "gpu_memory_used": 3980.0, - "relative_time": 5.386842727661133 + "timestamp": "2025-01-04T02:38:28.835603", + "cpu_percent": 26.81, + "ram_percent": 54.3, + "ram_used_gb": 34.536773681640625, + "gpu_memory_used": 4015.0, + "relative_time": 36.73981595039368 }, { - "timestamp": "2025-01-03T17:45:17.185921", - "cpu_percent": 13.14, - "ram_percent": 54.2, - "ram_used_gb": 34.4874153137207, - "gpu_memory_used": 3980.0, - "relative_time": 6.450911998748779 + "timestamp": "2025-01-04T02:38:29.912604", + "cpu_percent": 27.61, + "ram_percent": 54.3, + "ram_used_gb": 34.56916427612305, + "gpu_memory_used": 4016.0, + "relative_time": 37.81279993057251 }, { - "timestamp": "2025-01-03T17:45:18.248252", - "cpu_percent": 15.54, + "timestamp": "2025-01-04T02:38:30.984988", + "cpu_percent": 34.24, "ram_percent": 54.4, - "ram_used_gb": 34.581886291503906, - "gpu_memory_used": 3986.0, - "relative_time": 7.525278329849243 + "ram_used_gb": 34.599365234375, + "gpu_memory_used": 4014.0, + "relative_time": 38.89973425865173 }, { - "timestamp": "2025-01-03T17:45:19.324382", - "cpu_percent": 14.89, + "timestamp": "2025-01-04T02:38:32.071596", + "cpu_percent": 31.95, + "ram_percent": 54.2, + "ram_used_gb": 34.46506881713867, + "gpu_memory_used": 4014.0, + "relative_time": 39.95902729034424 + }, + { + "timestamp": "2025-01-04T02:38:33.140836", + "cpu_percent": 27.78, + "ram_percent": 54.3, + "ram_used_gb": 34.51242446899414, + "gpu_memory_used": 4014.0, + "relative_time": 41.0454580783844 + }, + { + "timestamp": "2025-01-04T02:38:34.229919", + "cpu_percent": 21.09, + "ram_percent": 54.3, + "ram_used_gb": 34.513973236083984, + "gpu_memory_used": 4011.0, + "relative_time": 42.133435010910034 + }, + { + "timestamp": "2025-01-04T02:38:35.317486", + "cpu_percent": 17.26, + "ram_percent": 53.9, + "ram_used_gb": 34.3167839050293, + "gpu_memory_used": 4020.0, + "relative_time": 43.21739077568054 + }, + { + "timestamp": "2025-01-04T02:38:36.394375", + "cpu_percent": 12.32, + "ram_percent": 54.0, + "ram_used_gb": 34.34043884277344, + "gpu_memory_used": 4020.0, + "relative_time": 44.27889919281006 + }, + { + "timestamp": "2025-01-04T02:38:37.454005", + "cpu_percent": 12.46, + "ram_percent": 54.0, + "ram_used_gb": 34.37453842163086, + "gpu_memory_used": 4020.0, + "relative_time": 45.341508626937866 + }, + { + "timestamp": "2025-01-04T02:38:38.515337", + "cpu_percent": 14.16, + "ram_percent": 54.1, + "ram_used_gb": 34.401729583740234, + "gpu_memory_used": 4019.0, + "relative_time": 46.410696506500244 + }, + { + "timestamp": "2025-01-04T02:38:39.593044", + "cpu_percent": 13.71, + "ram_percent": 54.1, + "ram_used_gb": 34.435630798339844, + "gpu_memory_used": 4019.0, + "relative_time": 47.48556661605835 + }, + { + "timestamp": "2025-01-04T02:38:40.665509", + "cpu_percent": 13.17, + "ram_percent": 54.2, + "ram_used_gb": 34.49795150756836, + "gpu_memory_used": 4016.0, + "relative_time": 48.551952838897705 + }, + { + "timestamp": "2025-01-04T02:38:41.724929", + "cpu_percent": 12.67, + "ram_percent": 54.3, + "ram_used_gb": 34.52568054199219, + "gpu_memory_used": 4011.0, + "relative_time": 49.61591196060181 + }, + { + "timestamp": "2025-01-04T02:38:42.801080", + "cpu_percent": 12.83, "ram_percent": 54.4, - "ram_used_gb": 34.5898551940918, - "gpu_memory_used": 3987.0, - "relative_time": 8.588879585266113 + "ram_used_gb": 34.579071044921875, + "gpu_memory_used": 4007.0, + "relative_time": 50.70357823371887 }, { - "timestamp": "2025-01-03T17:45:20.394701", - "cpu_percent": 12.13, + "timestamp": "2025-01-04T02:38:43.884984", + "cpu_percent": 12.31, "ram_percent": 54.4, - "ram_used_gb": 34.582420349121094, - "gpu_memory_used": 3986.0, - "relative_time": 9.65286660194397 + "ram_used_gb": 34.59829330444336, + "gpu_memory_used": 4003.0, + "relative_time": 51.771891832351685 }, { - "timestamp": "2025-01-03T17:45:21.455704", - "cpu_percent": 11.02, - "ram_percent": 54.4, - "ram_used_gb": 34.617252349853516, - "gpu_memory_used": 3986.0, - "relative_time": 10.71657395362854 - }, - { - "timestamp": "2025-01-03T17:45:22.525946", - "cpu_percent": 14.01, - "ram_percent": 54.5, - "ram_used_gb": 34.651466369628906, - "gpu_memory_used": 3989.0, - "relative_time": 11.787351846694946 - }, - { - "timestamp": "2025-01-03T17:45:23.584761", - "cpu_percent": 13.09, - "ram_percent": 54.5, - "ram_used_gb": 34.680885314941406, - "gpu_memory_used": 3989.0, - "relative_time": 12.846002101898193 - }, - { - "timestamp": "2025-01-03T17:45:24.645316", - "cpu_percent": 17.72, - "ram_percent": 54.6, - "ram_used_gb": 34.741127014160156, - "gpu_memory_used": 3985.0, - "relative_time": 13.918755054473877 - }, - { - "timestamp": "2025-01-03T17:45:25.718731", - "cpu_percent": 14.66, - "ram_percent": 54.6, - "ram_used_gb": 34.71047592163086, - "gpu_memory_used": 3982.0, - "relative_time": 14.974157810211182 - }, - { - "timestamp": "2025-01-03T17:45:26.774860", - "cpu_percent": 11.52, - "ram_percent": 54.6, - "ram_used_gb": 34.728397369384766, - "gpu_memory_used": 3982.0, - "relative_time": 16.034392833709717 - }, - { - "timestamp": "2025-01-03T17:45:27.837623", - "cpu_percent": 11.04, - "ram_percent": 54.6, - "ram_used_gb": 34.75224685668945, - "gpu_memory_used": 3981.0, - "relative_time": 17.096498250961304 - }, - { - "timestamp": "2025-01-03T17:45:28.898447", - "cpu_percent": 12.17, + "timestamp": "2025-01-04T02:38:44.957477", + "cpu_percent": 12.58, "ram_percent": 54.7, - "ram_used_gb": 34.796974182128906, - "gpu_memory_used": 3977.0, - "relative_time": 18.157397270202637 + "ram_used_gb": 34.76633071899414, + "gpu_memory_used": 4003.0, + "relative_time": 52.859192848205566 }, { - "timestamp": "2025-01-03T17:45:29.959510", - "cpu_percent": 10.72, + "timestamp": "2025-01-04T02:38:46.031581", + "cpu_percent": 14.48, + "ram_percent": 54.6, + "ram_used_gb": 34.76308059692383, + "gpu_memory_used": 4013.0, + "relative_time": 53.91648840904236 + }, + { + "timestamp": "2025-01-04T02:38:47.091693", + "cpu_percent": 14.35, "ram_percent": 54.7, - "ram_used_gb": 34.819969177246094, - "gpu_memory_used": 3991.0, - "relative_time": 19.22814679145813 + "ram_used_gb": 34.81193923950195, + "gpu_memory_used": 4013.0, + "relative_time": 54.993882179260254 }, { - "timestamp": "2025-01-03T17:45:31.033262", - "cpu_percent": 17.95, - "ram_percent": 55.0, - "ram_used_gb": 34.9871711730957, - "gpu_memory_used": 3995.0, - "relative_time": 20.29205060005188 + "timestamp": "2025-01-04T02:38:48.178826", + "cpu_percent": 16.46, + "ram_percent": 54.7, + "ram_used_gb": 34.784278869628906, + "gpu_memory_used": 4014.0, + "relative_time": 56.064146518707275 }, { - "timestamp": "2025-01-03T17:45:32.091757", - "cpu_percent": 19.11, - "ram_percent": 55.0, - "ram_used_gb": 35.0067138671875, - "gpu_memory_used": 3995.0, - "relative_time": 21.353832006454468 + "timestamp": "2025-01-04T02:38:49.235997", + "cpu_percent": 12.84, + "ram_percent": 54.7, + "ram_used_gb": 34.79767608642578, + "gpu_memory_used": 4014.0, + "relative_time": 57.12374472618103 }, { - "timestamp": "2025-01-03T17:45:33.156831", - "cpu_percent": 32.93, + "timestamp": "2025-01-04T02:38:50.295962", + "cpu_percent": 15.69, + "ram_percent": 54.8, + "ram_used_gb": 34.8546257019043, + "gpu_memory_used": 4013.0, + "relative_time": 58.180296421051025 + }, + { + "timestamp": "2025-01-04T02:38:51.357678", + "cpu_percent": 14.54, + "ram_percent": 54.8, + "ram_used_gb": 34.8900260925293, + "gpu_memory_used": 4014.0, + "relative_time": 59.242270708084106 + }, + { + "timestamp": "2025-01-04T02:38:52.415380", + "cpu_percent": 14.74, + "ram_percent": 54.9, + "ram_used_gb": 34.92173767089844, + "gpu_memory_used": 4014.0, + "relative_time": 60.307114601135254 + }, + { + "timestamp": "2025-01-04T02:38:53.490598", + "cpu_percent": 13.82, "ram_percent": 55.1, - "ram_used_gb": 35.05879211425781, - "gpu_memory_used": 3995.0, - "relative_time": 22.416496992111206 + "ram_used_gb": 35.028907775878906, + "gpu_memory_used": 4014.0, + "relative_time": 61.37576389312744 }, { - "timestamp": "2025-01-03T17:45:34.217136", - "cpu_percent": 12.59, + "timestamp": "2025-01-04T02:38:54.548660", + "cpu_percent": 11.31, + "ram_percent": 55.1, + "ram_used_gb": 35.05375289916992, + "gpu_memory_used": 4011.0, + "relative_time": 62.43392610549927 + }, + { + "timestamp": "2025-01-04T02:38:55.609900", + "cpu_percent": 14.35, + "ram_percent": 55.1, + "ram_used_gb": 35.03831100463867, + "gpu_memory_used": 4011.0, + "relative_time": 63.493370056152344 + }, + { + "timestamp": "2025-01-04T02:38:56.666032", + "cpu_percent": 13.11, + "ram_percent": 55.1, + "ram_used_gb": 35.07795333862305, + "gpu_memory_used": 4011.0, + "relative_time": 64.54955720901489 + }, + { + "timestamp": "2025-01-04T02:38:57.730782", + "cpu_percent": 16.01, "ram_percent": 55.2, - "ram_used_gb": 35.10686111450195, - "gpu_memory_used": 3994.0, - "relative_time": 23.476072549819946 + "ram_used_gb": 35.11598587036133, + "gpu_memory_used": 4007.0, + "relative_time": 65.61445665359497 }, { - "timestamp": "2025-01-03T17:45:35.275577", - "cpu_percent": 30.79, + "timestamp": "2025-01-04T02:38:58.787051", + "cpu_percent": 13.68, + "ram_percent": 55.3, + "ram_used_gb": 35.15185546875, + "gpu_memory_used": 4007.0, + "relative_time": 66.67095923423767 + }, + { + "timestamp": "2025-01-04T02:38:59.847606", + "cpu_percent": 13.08, + "ram_percent": 55.3, + "ram_used_gb": 35.183753967285156, + "gpu_memory_used": 4003.0, + "relative_time": 67.73307466506958 + }, + { + "timestamp": "2025-01-04T02:39:00.918871", + "cpu_percent": 13.26, "ram_percent": 55.4, - "ram_used_gb": 35.22132110595703, - "gpu_memory_used": 3989.0, - "relative_time": 24.564188957214355 + "ram_used_gb": 35.22275161743164, + "gpu_memory_used": 4005.0, + "relative_time": 68.80590057373047 }, { - "timestamp": "2025-01-03T17:45:36.365095", - "cpu_percent": 13.36, - "ram_percent": 55.2, - "ram_used_gb": 35.08255386352539, - "gpu_memory_used": 4000.0, - "relative_time": 25.64090871810913 + "timestamp": "2025-01-04T02:39:01.991170", + "cpu_percent": 11.05, + "ram_percent": 55.3, + "ram_used_gb": 35.15507125854492, + "gpu_memory_used": 4005.0, + "relative_time": 69.8813705444336 }, { - "timestamp": "2025-01-03T17:45:37.451539", - "cpu_percent": 14.94, - "ram_percent": 55.2, - "ram_used_gb": 35.118614196777344, - "gpu_memory_used": 4000.0, - "relative_time": 26.71500325202942 - }, - { - "timestamp": "2025-01-03T17:45:38.525364", - "cpu_percent": 12.76, + "timestamp": "2025-01-04T02:39:03.058942", + "cpu_percent": 11.08, "ram_percent": 55.4, - "ram_used_gb": 35.221614837646484, - "gpu_memory_used": 3999.0, - "relative_time": 27.806236505508423 + "ram_used_gb": 35.2095947265625, + "gpu_memory_used": 4005.0, + "relative_time": 70.94484400749207 }, { - "timestamp": "2025-01-03T17:45:39.616790", - "cpu_percent": 16.11, - "ram_percent": 55.4, - "ram_used_gb": 35.2247200012207, - "gpu_memory_used": 3999.0, - "relative_time": 28.875747203826904 - }, - { - "timestamp": "2025-01-03T17:45:40.675234", - "cpu_percent": 14.96, - "ram_percent": 55.4, - "ram_used_gb": 35.21339416503906, - "gpu_memory_used": 3999.0, - "relative_time": 29.94703769683838 - }, - { - "timestamp": "2025-01-03T17:45:41.746176", - "cpu_percent": 10.99, - "ram_percent": 55.4, - "ram_used_gb": 35.260677337646484, - "gpu_memory_used": 3994.0, - "relative_time": 31.006144046783447 - }, - { - "timestamp": "2025-01-03T17:45:42.807809", - "cpu_percent": 13.15, - "ram_percent": 55.5, - "ram_used_gb": 35.299591064453125, - "gpu_memory_used": 3994.0, - "relative_time": 32.0741171836853 - }, - { - "timestamp": "2025-01-03T17:45:43.879826", + "timestamp": "2025-01-04T02:39:04.116159", "cpu_percent": 12.74, + "ram_percent": 55.4, + "ram_used_gb": 35.24392318725586, + "gpu_memory_used": 4005.0, + "relative_time": 72.00281810760498 + }, + { + "timestamp": "2025-01-04T02:39:05.187410", + "cpu_percent": 11.04, + "ram_percent": 55.4, + "ram_used_gb": 35.259830474853516, + "gpu_memory_used": 4005.0, + "relative_time": 73.07217526435852 + }, + { + "timestamp": "2025-01-04T02:39:06.248588", + "cpu_percent": 13.66, + "ram_percent": 55.5, + "ram_used_gb": 35.29854965209961, + "gpu_memory_used": 4005.0, + "relative_time": 74.13533973693848 + }, + { + "timestamp": "2025-01-04T02:39:07.321179", + "cpu_percent": 11.16, "ram_percent": 55.6, - "ram_used_gb": 35.34665298461914, - "gpu_memory_used": 3994.0, - "relative_time": 33.14525270462036 + "ram_used_gb": 35.346981048583984, + "gpu_memory_used": 4005.0, + "relative_time": 75.18772435188293 }, { - "timestamp": "2025-01-03T17:45:44.954413", - "cpu_percent": 12.11, - "ram_percent": 55.6, - "ram_used_gb": 35.34089660644531, - "gpu_memory_used": 3990.0, - "relative_time": 34.21659064292908 + "timestamp": "2025-01-04T02:39:08.362496", + "cpu_percent": 9.46, + "ram_percent": 56.1, + "ram_used_gb": 35.69393539428711, + "gpu_memory_used": 4006.0, + "relative_time": 76.25136637687683 }, { - "timestamp": "2025-01-03T17:45:46.025229", - "cpu_percent": 13.02, - "ram_percent": 55.6, - "ram_used_gb": 35.37482833862305, - "gpu_memory_used": 3991.0, - "relative_time": 35.28446078300476 + "timestamp": "2025-01-04T02:39:09.438218", + "cpu_percent": 15.88, + "ram_percent": 56.1, + "ram_used_gb": 35.6658821105957, + "gpu_memory_used": 4006.0, + "relative_time": 77.32329249382019 }, { - "timestamp": "2025-01-03T17:45:47.085470", - "cpu_percent": 13.53, - "ram_percent": 55.6, - "ram_used_gb": 35.392356872558594, - "gpu_memory_used": 3988.0, - "relative_time": 36.34242486953735 - }, - { - "timestamp": "2025-01-03T17:45:48.155295", - "cpu_percent": 15.0, - "ram_percent": 55.7, - "ram_used_gb": 35.449764251708984, - "gpu_memory_used": 3987.0, - "relative_time": 37.418004512786865 - }, - { - "timestamp": "2025-01-03T17:45:49.218400", - "cpu_percent": 13.84, - "ram_percent": 55.8, - "ram_used_gb": 35.468841552734375, - "gpu_memory_used": 3986.0, - "relative_time": 38.48085808753967 - }, - { - "timestamp": "2025-01-03T17:45:50.281360", + "timestamp": "2025-01-04T02:39:10.495653", "cpu_percent": 13.25, - "ram_percent": 55.8, - "ram_used_gb": 35.491825103759766, - "gpu_memory_used": 3987.0, - "relative_time": 39.5399751663208 + "ram_percent": 56.1, + "ram_used_gb": 35.658119201660156, + "gpu_memory_used": 4014.0, + "relative_time": 78.38210940361023 }, { - "timestamp": "2025-01-03T17:45:51.343810", - "cpu_percent": 10.34, - "ram_percent": 55.8, - "ram_used_gb": 35.51161193847656, - "gpu_memory_used": 3985.0, - "relative_time": 40.60230302810669 + "timestamp": "2025-01-04T02:39:11.560094", + "cpu_percent": 10.8, + "ram_percent": 56.1, + "ram_used_gb": 35.694610595703125, + "gpu_memory_used": 4014.0, + "relative_time": 79.44816374778748 }, { - "timestamp": "2025-01-03T17:45:52.402527", - "cpu_percent": 12.56, - "ram_percent": 55.9, - "ram_used_gb": 35.57502365112305, - "gpu_memory_used": 3984.0, - "relative_time": 41.660725116729736 + "timestamp": "2025-01-04T02:39:12.621879", + "cpu_percent": 12.58, + "ram_percent": 56.1, + "ram_used_gb": 35.68545150756836, + "gpu_memory_used": 4014.0, + "relative_time": 80.51017951965332 }, { - "timestamp": "2025-01-03T17:45:53.460932", - "cpu_percent": 12.04, - "ram_percent": 56.0, - "ram_used_gb": 35.61081314086914, - "gpu_memory_used": 3978.0, - "relative_time": 42.71787190437317 + "timestamp": "2025-01-04T02:39:13.692834", + "cpu_percent": 13.3, + "ram_percent": 56.2, + "ram_used_gb": 35.730979919433594, + "gpu_memory_used": 4012.0, + "relative_time": 81.57789969444275 }, { - "timestamp": "2025-01-03T17:45:54.521959", - "cpu_percent": 10.13, + "timestamp": "2025-01-04T02:39:14.753401", + "cpu_percent": 14.75, + "ram_percent": 56.2, + "ram_used_gb": 35.73103332519531, + "gpu_memory_used": 4012.0, + "relative_time": 82.63830900192261 + }, + { + "timestamp": "2025-01-04T02:39:15.811385", + "cpu_percent": 14.69, + "ram_percent": 56.2, + "ram_used_gb": 35.740108489990234, + "gpu_memory_used": 4011.0, + "relative_time": 83.69796371459961 + }, + { + "timestamp": "2025-01-04T02:39:16.874197", + "cpu_percent": 14.28, + "ram_percent": 56.2, + "ram_used_gb": 35.767982482910156, + "gpu_memory_used": 4010.0, + "relative_time": 84.76145887374878 + }, + { + "timestamp": "2025-01-04T02:39:17.936944", + "cpu_percent": 12.99, "ram_percent": 56.3, - "ram_used_gb": 35.822574615478516, - "gpu_memory_used": 3978.0, - "relative_time": 43.783926010131836 + "ram_used_gb": 35.81233596801758, + "gpu_memory_used": 4010.0, + "relative_time": 85.84119439125061 }, { - "timestamp": "2025-01-03T17:45:55.583212", - "cpu_percent": 28.17, - "ram_percent": 56.3, - "ram_used_gb": 35.78395462036133, - "gpu_memory_used": 3976.0, - "relative_time": 44.858543157577515 - }, - { - "timestamp": "2025-01-03T17:45:56.657026", - "cpu_percent": 16.61, - "ram_percent": 56.3, - "ram_used_gb": 35.7921028137207, - "gpu_memory_used": 3984.0, - "relative_time": 45.918612003326416 - }, - { - "timestamp": "2025-01-03T17:45:57.716203", - "cpu_percent": 15.03, - "ram_percent": 56.3, - "ram_used_gb": 35.79140853881836, - "gpu_memory_used": 3984.0, - "relative_time": 46.97588872909546 - }, - { - "timestamp": "2025-01-03T17:45:58.775392", - "cpu_percent": 14.81, - "ram_percent": 56.3, - "ram_used_gb": 35.80635452270508, - "gpu_memory_used": 3984.0, - "relative_time": 48.03421711921692 - }, - { - "timestamp": "2025-01-03T17:45:59.834277", - "cpu_percent": 15.06, - "ram_percent": 56.3, - "ram_used_gb": 35.81984329223633, - "gpu_memory_used": 3984.0, - "relative_time": 49.0965371131897 - }, - { - "timestamp": "2025-01-03T17:46:00.896761", - "cpu_percent": 19.76, - "ram_percent": 56.3, - "ram_used_gb": 35.7983512878418, - "gpu_memory_used": 3989.0, - "relative_time": 50.177143812179565 - }, - { - "timestamp": "2025-01-03T17:46:01.981868", - "cpu_percent": 17.32, - "ram_percent": 56.3, - "ram_used_gb": 35.81730270385742, - "gpu_memory_used": 3990.0, - "relative_time": 51.242098331451416 - }, - { - "timestamp": "2025-01-03T17:46:03.046930", - "cpu_percent": 19.8, - "ram_percent": 56.5, - "ram_used_gb": 35.92729949951172, - "gpu_memory_used": 3990.0, - "relative_time": 52.3223512172699 - }, - { - "timestamp": "2025-01-03T17:46:04.122311", - "cpu_percent": 20.91, - "ram_percent": 56.5, - "ram_used_gb": 35.949684143066406, - "gpu_memory_used": 3991.0, - "relative_time": 53.3851900100708 - }, - { - "timestamp": "2025-01-03T17:46:05.182768", - "cpu_percent": 17.39, - "ram_percent": 56.5, - "ram_used_gb": 35.94847869873047, - "gpu_memory_used": 3991.0, - "relative_time": 54.45881199836731 - }, - { - "timestamp": "2025-01-03T17:46:06.257550", - "cpu_percent": 16.64, - "ram_percent": 56.5, - "ram_used_gb": 35.9198112487793, - "gpu_memory_used": 3989.0, - "relative_time": 55.51820731163025 - }, - { - "timestamp": "2025-01-03T17:46:07.317263", - "cpu_percent": 15.99, - "ram_percent": 56.3, - "ram_used_gb": 35.82686233520508, - "gpu_memory_used": 3989.0, - "relative_time": 56.59837555885315 - }, - { - "timestamp": "2025-01-03T17:46:08.409244", - "cpu_percent": 15.11, - "ram_percent": 56.4, - "ram_used_gb": 35.852657318115234, - "gpu_memory_used": 3988.0, - "relative_time": 57.669328927993774 - }, - { - "timestamp": "2025-01-03T17:46:09.473703", - "cpu_percent": 18.54, - "ram_percent": 56.4, - "ram_used_gb": 35.889339447021484, - "gpu_memory_used": 3979.0, - "relative_time": 58.76238036155701 - }, - { - "timestamp": "2025-01-03T17:46:10.562180", - "cpu_percent": 15.7, - "ram_percent": 56.4, - "ram_used_gb": 35.90079879760742, - "gpu_memory_used": 3975.0, - "relative_time": 59.82209253311157 - }, - { - "timestamp": "2025-01-03T17:46:11.634373", - "cpu_percent": 16.25, - "ram_percent": 56.5, - "ram_used_gb": 35.94197082519531, - "gpu_memory_used": 3976.0, - "relative_time": 60.91385841369629 - }, - { - "timestamp": "2025-01-03T17:46:12.723458", - "cpu_percent": 16.98, + "timestamp": "2025-01-04T02:39:19.027320", + "cpu_percent": 12.79, "ram_percent": 56.6, - "ram_used_gb": 35.99095153808594, - "gpu_memory_used": 3976.0, - "relative_time": 61.981855154037476 + "ram_used_gb": 36.0085563659668, + "gpu_memory_used": 4010.0, + "relative_time": 86.91442775726318 }, { - "timestamp": "2025-01-03T17:46:13.781955", - "cpu_percent": 15.59, - "ram_percent": 56.6, - "ram_used_gb": 36.00953674316406, - "gpu_memory_used": 3976.0, - "relative_time": 63.04051613807678 + "timestamp": "2025-01-04T02:39:20.088804", + "cpu_percent": 15.62, + "ram_percent": 56.9, + "ram_used_gb": 36.16616439819336, + "gpu_memory_used": 4006.0, + "relative_time": 88.00494360923767 }, { - "timestamp": "2025-01-03T17:46:14.852706", - "cpu_percent": 13.16, - "ram_percent": 56.7, - "ram_used_gb": 36.050899505615234, - "gpu_memory_used": 3976.0, - "relative_time": 64.11573505401611 + "timestamp": "2025-01-04T02:39:21.191796", + "cpu_percent": 12.98, + "ram_percent": 57.1, + "ram_used_gb": 36.3217658996582, + "gpu_memory_used": 4005.0, + "relative_time": 89.12522411346436 }, { - "timestamp": "2025-01-03T17:46:15.927719", - "cpu_percent": 12.34, - "ram_percent": 56.7, - "ram_used_gb": 36.07988739013672, - "gpu_memory_used": 3976.0, - "relative_time": 65.18661308288574 + "timestamp": "2025-01-04T02:39:22.311508", + "cpu_percent": 14.77, + "ram_percent": 56.9, + "ram_used_gb": 36.191429138183594, + "gpu_memory_used": 4005.0, + "relative_time": 90.19932198524475 }, { - "timestamp": "2025-01-03T17:46:16.999292", - "cpu_percent": 12.34, - "ram_percent": 56.8, - "ram_used_gb": 36.099937438964844, - "gpu_memory_used": 3976.0, - "relative_time": 66.25790786743164 + "timestamp": "2025-01-04T02:39:23.372871", + "cpu_percent": 12.98, + "ram_percent": 57.1, + "ram_used_gb": 36.29658889770508, + "gpu_memory_used": 4005.0, + "relative_time": 91.3045928478241 }, { - "timestamp": "2025-01-03T17:46:18.058608", + "timestamp": "2025-01-04T02:39:24.479087", + "cpu_percent": 14.64, + "ram_percent": 57.1, + "ram_used_gb": 36.30413055419922, + "gpu_memory_used": 3998.0, + "relative_time": 92.36360597610474 + }, + { + "timestamp": "2025-01-04T02:39:25.537969", + "cpu_percent": 14.51, + "ram_percent": 57.1, + "ram_used_gb": 36.311763763427734, + "gpu_memory_used": 3998.0, + "relative_time": 93.42230415344238 + }, + { + "timestamp": "2025-01-04T02:39:26.594967", + "cpu_percent": 13.05, + "ram_percent": 57.1, + "ram_used_gb": 36.351402282714844, + "gpu_memory_used": 3998.0, + "relative_time": 94.47847175598145 + }, + { + "timestamp": "2025-01-04T02:39:27.652223", + "cpu_percent": 15.05, + "ram_percent": 57.2, + "ram_used_gb": 36.36949157714844, + "gpu_memory_used": 4004.0, + "relative_time": 95.53560948371887 + }, + { + "timestamp": "2025-01-04T02:39:28.708008", + "cpu_percent": 12.51, + "ram_percent": 57.2, + "ram_used_gb": 36.3841667175293, + "gpu_memory_used": 4004.0, + "relative_time": 96.59472155570984 + }, + { + "timestamp": "2025-01-04T02:39:29.768866", + "cpu_percent": 10.83, + "ram_percent": 57.2, + "ram_used_gb": 36.39939880371094, + "gpu_memory_used": 4004.0, + "relative_time": 97.6679356098175 + }, + { + "timestamp": "2025-01-04T02:39:30.844295", + "cpu_percent": 14.61, + "ram_percent": 57.3, + "ram_used_gb": 36.42519760131836, + "gpu_memory_used": 4004.0, + "relative_time": 98.74996089935303 + }, + { + "timestamp": "2025-01-04T02:39:31.934080", "cpu_percent": 11.74, - "ram_percent": 56.8, - "ram_used_gb": 36.14547348022461, - "gpu_memory_used": 3975.0, - "relative_time": 67.31676268577576 + "ram_percent": 57.0, + "ram_used_gb": 36.271087646484375, + "gpu_memory_used": 4004.0, + "relative_time": 99.81860518455505 }, { - "timestamp": "2025-01-03T17:46:19.122597", - "cpu_percent": 12.63, - "ram_percent": 56.9, - "ram_used_gb": 36.177284240722656, - "gpu_memory_used": 3974.0, - "relative_time": 68.3815085887909 + "timestamp": "2025-01-04T02:39:32.989954", + "cpu_percent": 12.09, + "ram_percent": 57.2, + "ram_used_gb": 36.368350982666016, + "gpu_memory_used": 4010.0, + "relative_time": 100.87712931632996 }, { - "timestamp": "2025-01-03T17:46:20.182864", - "cpu_percent": 9.65, - "ram_percent": 56.9, - "ram_used_gb": 36.216495513916016, - "gpu_memory_used": 3973.0, - "relative_time": 69.44507431983948 + "timestamp": "2025-01-04T02:39:34.061411", + "cpu_percent": 11.07, + "ram_percent": 57.2, + "ram_used_gb": 36.38072967529297, + "gpu_memory_used": 4010.0, + "relative_time": 101.946035861969 }, { - "timestamp": "2025-01-03T17:46:21.244696", - "cpu_percent": 10.38, + "timestamp": "2025-01-04T02:39:35.117182", + "cpu_percent": 9.32, + "ram_percent": 57.2, + "ram_used_gb": 36.367733001708984, + "gpu_memory_used": 4415.0, + "relative_time": 103.00355505943298 + }, + { + "timestamp": "2025-01-04T02:39:36.179256", + "cpu_percent": 12.93, + "ram_percent": 57.2, + "ram_used_gb": 36.396636962890625, + "gpu_memory_used": 4417.0, + "relative_time": 104.06347131729126 + }, + { + "timestamp": "2025-01-04T02:39:37.237454", + "cpu_percent": 10.94, + "ram_percent": 57.3, + "ram_used_gb": 36.429630279541016, + "gpu_memory_used": 4417.0, + "relative_time": 105.12580728530884 + }, + { + "timestamp": "2025-01-04T02:39:38.310321", + "cpu_percent": 12.86, + "ram_percent": 57.3, + "ram_used_gb": 36.44291305541992, + "gpu_memory_used": 4418.0, + "relative_time": 106.17753839492798 + }, + { + "timestamp": "2025-01-04T02:39:39.355358", + "cpu_percent": 10.82, + "ram_percent": 57.3, + "ram_used_gb": 36.46603012084961, + "gpu_memory_used": 4418.0, + "relative_time": 107.24251008033752 + }, + { + "timestamp": "2025-01-04T02:39:40.413524", + "cpu_percent": 14.64, "ram_percent": 57.4, - "ram_used_gb": 36.51596450805664, - "gpu_memory_used": 3973.0, - "relative_time": 70.51762080192566 + "ram_used_gb": 36.507179260253906, + "gpu_memory_used": 4418.0, + "relative_time": 108.29774165153503 }, { - "timestamp": "2025-01-03T17:46:22.448455", - "cpu_percent": 9.24, - "ram_percent": 57.5, - "ram_used_gb": 36.56745529174805, - "gpu_memory_used": 3974.0, - "relative_time": 71.72753357887268 + "timestamp": "2025-01-04T02:39:41.482368", + "cpu_percent": 10.03, + "ram_percent": 58.1, + "ram_used_gb": 36.93812942504883, + "gpu_memory_used": 4418.0, + "relative_time": 109.36836910247803 + }, + { + "timestamp": "2025-01-04T02:39:42.546204", + "cpu_percent": 12.63, + "ram_percent": 58.0, + "ram_used_gb": 36.87542724609375, + "gpu_memory_used": 4418.0, + "relative_time": 110.43055510520935 + }, + { + "timestamp": "2025-01-04T02:39:43.604666", + "cpu_percent": 14.14, + "ram_percent": 58.0, + "ram_used_gb": 36.875328063964844, + "gpu_memory_used": 4426.0, + "relative_time": 111.49229407310486 + }, + { + "timestamp": "2025-01-04T02:39:44.664973", + "cpu_percent": 10.64, + "ram_percent": 58.0, + "ram_used_gb": 36.88217544555664, + "gpu_memory_used": 4425.0, + "relative_time": 112.55481696128845 + }, + { + "timestamp": "2025-01-04T02:39:45.741305", + "cpu_percent": 13.92, + "ram_percent": 57.9, + "ram_used_gb": 36.85449981689453, + "gpu_memory_used": 4425.0, + "relative_time": 113.62504053115845 + }, + { + "timestamp": "2025-01-04T02:39:46.799656", + "cpu_percent": 13.15, + "ram_percent": 58.0, + "ram_used_gb": 36.870826721191406, + "gpu_memory_used": 4423.0, + "relative_time": 114.6845052242279 + }, + { + "timestamp": "2025-01-04T02:39:47.859725", + "cpu_percent": 13.85, + "ram_percent": 58.0, + "ram_used_gb": 36.870948791503906, + "gpu_memory_used": 4423.0, + "relative_time": 115.74664235115051 + }, + { + "timestamp": "2025-01-04T02:39:48.919071", + "cpu_percent": 14.59, + "ram_percent": 58.0, + "ram_used_gb": 36.886802673339844, + "gpu_memory_used": 4422.0, + "relative_time": 116.80267906188965 + }, + { + "timestamp": "2025-01-04T02:39:49.976541", + "cpu_percent": 16.56, + "ram_percent": 58.0, + "ram_used_gb": 36.90068435668945, + "gpu_memory_used": 4422.0, + "relative_time": 117.86520886421204 + }, + { + "timestamp": "2025-01-04T02:39:51.036593", + "cpu_percent": 10.33, + "ram_percent": 58.1, + "ram_used_gb": 36.96821212768555, + "gpu_memory_used": 4416.0, + "relative_time": 118.92232513427734 + }, + { + "timestamp": "2025-01-04T02:39:52.098186", + "cpu_percent": 15.23, + "ram_percent": 58.1, + "ram_used_gb": 36.96358108520508, + "gpu_memory_used": 4416.0, + "relative_time": 119.98378920555115 + }, + { + "timestamp": "2025-01-04T02:39:53.168566", + "cpu_percent": 11.96, + "ram_percent": 58.2, + "ram_used_gb": 37.00669479370117, + "gpu_memory_used": 4416.0, + "relative_time": 121.05223441123962 + }, + { + "timestamp": "2025-01-04T02:39:54.230292", + "cpu_percent": 13.03, + "ram_percent": 58.2, + "ram_used_gb": 37.031307220458984, + "gpu_memory_used": 4416.0, + "relative_time": 122.11563086509705 + }, + { + "timestamp": "2025-01-04T02:39:55.287670", + "cpu_percent": 14.93, + "ram_percent": 58.2, + "ram_used_gb": 37.045589447021484, + "gpu_memory_used": 4416.0, + "relative_time": 123.17237305641174 + }, + { + "timestamp": "2025-01-04T02:39:56.349816", + "cpu_percent": 12.91, + "ram_percent": 58.3, + "ram_used_gb": 37.072689056396484, + "gpu_memory_used": 4416.0, + "relative_time": 124.23265671730042 + }, + { + "timestamp": "2025-01-04T02:39:57.409384", + "cpu_percent": 11.35, + "ram_percent": 58.3, + "ram_used_gb": 37.10455322265625, + "gpu_memory_used": 4416.0, + "relative_time": 125.29221749305725 + }, + { + "timestamp": "2025-01-04T02:39:58.464653", + "cpu_percent": 12.97, + "ram_percent": 58.4, + "ram_used_gb": 37.12955093383789, + "gpu_memory_used": 4416.0, + "relative_time": 126.34849739074707 + }, + { + "timestamp": "2025-01-04T02:39:59.521737", + "cpu_percent": 11.69, + "ram_percent": 58.4, + "ram_used_gb": 37.139190673828125, + "gpu_memory_used": 4416.0, + "relative_time": 127.40602111816406 + }, + { + "timestamp": "2025-01-04T02:40:00.581455", + "cpu_percent": 12.86, + "ram_percent": 58.5, + "ram_used_gb": 37.204673767089844, + "gpu_memory_used": 4418.0, + "relative_time": 128.5798671245575 + }, + { + "timestamp": "2025-01-04T02:40:01.760893", + "cpu_percent": 81.59, + "ram_percent": 58.1, + "ram_used_gb": 36.97315216064453, + "gpu_memory_used": 4425.0, + "relative_time": 129.6643455028534 + }, + { + "timestamp": "2025-01-04T02:40:02.850000", + "cpu_percent": 14.55, + "ram_percent": 58.2, + "ram_used_gb": 37.0354118347168, + "gpu_memory_used": 4435.0, + "relative_time": 130.7529788017273 + }, + { + "timestamp": "2025-01-04T02:40:03.934586", + "cpu_percent": 13.16, + "ram_percent": 58.2, + "ram_used_gb": 37.03787612915039, + "gpu_memory_used": 4437.0, + "relative_time": 131.81812405586243 + }, + { + "timestamp": "2025-01-04T02:40:04.989915", + "cpu_percent": 11.0, + "ram_percent": 58.3, + "ram_used_gb": 37.09538650512695, + "gpu_memory_used": 4437.0, + "relative_time": 132.88244915008545 + }, + { + "timestamp": "2025-01-04T02:40:06.067559", + "cpu_percent": 32.79, + "ram_percent": 58.3, + "ram_used_gb": 37.07184982299805, + "gpu_memory_used": 4437.0, + "relative_time": 133.97513842582703 + }, + { + "timestamp": "2025-01-04T02:40:07.156881", + "cpu_percent": 22.34, + "ram_percent": 58.3, + "ram_used_gb": 37.07517623901367, + "gpu_memory_used": 4438.0, + "relative_time": 135.04176831245422 + }, + { + "timestamp": "2025-01-04T02:40:08.212791", + "cpu_percent": 14.86, + "ram_percent": 58.3, + "ram_used_gb": 37.08013153076172, + "gpu_memory_used": 4438.0, + "relative_time": 136.1427457332611 + }, + { + "timestamp": "2025-01-04T02:40:09.317884", + "cpu_percent": 21.55, + "ram_percent": 58.4, + "ram_used_gb": 37.144142150878906, + "gpu_memory_used": 4447.0, + "relative_time": 137.20455360412598 + }, + { + "timestamp": "2025-01-04T02:40:10.390292", + "cpu_percent": 26.97, + "ram_percent": 58.4, + "ram_used_gb": 37.141868591308594, + "gpu_memory_used": 4454.0, + "relative_time": 138.2930736541748 + }, + { + "timestamp": "2025-01-04T02:40:11.464548", + "cpu_percent": 12.21, + "ram_percent": 58.5, + "ram_used_gb": 37.205867767333984, + "gpu_memory_used": 4451.0, + "relative_time": 139.35198616981506 + }, + { + "timestamp": "2025-01-04T02:40:12.537687", + "cpu_percent": 13.14, + "ram_percent": 58.5, + "ram_used_gb": 37.233299255371094, + "gpu_memory_used": 4452.0, + "relative_time": 140.4236707687378 + }, + { + "timestamp": "2025-01-04T02:40:13.608158", + "cpu_percent": 13.93, + "ram_percent": 58.6, + "ram_used_gb": 37.25624465942383, + "gpu_memory_used": 4452.0, + "relative_time": 141.4932518005371 + }, + { + "timestamp": "2025-01-04T02:40:14.668199", + "cpu_percent": 15.76, + "ram_percent": 58.6, + "ram_used_gb": 37.278499603271484, + "gpu_memory_used": 4452.0, + "relative_time": 142.57055759429932 + }, + { + "timestamp": "2025-01-04T02:40:15.754506", + "cpu_percent": 14.59, + "ram_percent": 58.5, + "ram_used_gb": 37.21017837524414, + "gpu_memory_used": 4451.0, + "relative_time": 143.64187097549438 + }, + { + "timestamp": "2025-01-04T02:40:16.827392", + "cpu_percent": 17.55, + "ram_percent": 58.4, + "ram_used_gb": 37.1205940246582, + "gpu_memory_used": 4450.0, + "relative_time": 144.75147438049316 + }, + { + "timestamp": "2025-01-04T02:40:17.929858", + "cpu_percent": 55.09, + "ram_percent": 58.2, + "ram_used_gb": 37.04216766357422, + "gpu_memory_used": 4449.0, + "relative_time": 145.815936088562 + }, + { + "timestamp": "2025-01-04T02:40:18.988009", + "cpu_percent": 13.92, + "ram_percent": 58.0, + "ram_used_gb": 36.90068054199219, + "gpu_memory_used": 4447.0, + "relative_time": 146.8880341053009 + }, + { + "timestamp": "2025-01-04T02:40:20.062567", + "cpu_percent": 17.42, + "ram_percent": 57.7, + "ram_used_gb": 36.69455337524414, + "gpu_memory_used": 4446.0, + "relative_time": 147.96440315246582 + }, + { + "timestamp": "2025-01-04T02:40:21.149129", + "cpu_percent": 14.78, + "ram_percent": 57.4, + "ram_used_gb": 36.50687789916992, + "gpu_memory_used": 4445.0, + "relative_time": 149.041100025177 + }, + { + "timestamp": "2025-01-04T02:40:22.221780", + "cpu_percent": 11.1, + "ram_percent": 57.0, + "ram_used_gb": 36.28267288208008, + "gpu_memory_used": 4438.0, + "relative_time": 150.125506401062 + }, + { + "timestamp": "2025-01-04T02:40:23.308492", + "cpu_percent": 12.21, + "ram_percent": 56.7, + "ram_used_gb": 36.036773681640625, + "gpu_memory_used": 4436.0, + "relative_time": 151.19524502754211 + }, + { + "timestamp": "2025-01-04T02:40:24.381177", + "cpu_percent": 13.79, + "ram_percent": 56.3, + "ram_used_gb": 35.83684539794922, + "gpu_memory_used": 4436.0, + "relative_time": 152.26534175872803 + }, + { + "timestamp": "2025-01-04T02:40:25.452457", + "cpu_percent": 12.28, + "ram_percent": 56.4, + "ram_used_gb": 35.848087310791016, + "gpu_memory_used": 4436.0, + "relative_time": 153.33880996704102 + }, + { + "timestamp": "2025-01-04T02:40:26.521613", + "cpu_percent": 12.52, + "ram_percent": 56.8, + "ram_used_gb": 36.1606330871582, + "gpu_memory_used": 4440.0, + "relative_time": 154.40920901298523 + }, + { + "timestamp": "2025-01-04T02:40:27.587547", + "cpu_percent": 7.94, + "ram_percent": 57.2, + "ram_used_gb": 36.37208557128906, + "gpu_memory_used": 4440.0, + "relative_time": 155.46942234039307 + }, + { + "timestamp": "2025-01-04T02:40:28.647400", + "cpu_percent": 8.85, + "ram_percent": 57.3, + "ram_used_gb": 36.470054626464844, + "gpu_memory_used": 4440.0, + "relative_time": 156.53129720687866 } ], - "test_duration": 74.18872809410095 + "test_duration": 159.19756031036377 } \ No newline at end of file diff --git a/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_stats_rtf.txt b/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_stats_rtf.txt index b045833..cb2df6a 100644 --- a/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_stats_rtf.txt +++ b/examples/assorted_checks/benchmarks/output_data/gpu_benchmark_stats_rtf.txt @@ -1,23 +1,23 @@ === Benchmark Statistics (with correct RTF) === -Total tokens processed: 8150 -Total audio generated (s): 2549.70 -Total test duration (s): 70.70 -Average processing rate (tokens/s): 120.20 +Total tokens processed: 17150 +Total audio generated (s): 5296.38 +Total test duration (s): 155.23 +Average processing rate (tokens/s): 102.86 Average RTF: 0.03 -Average Real Time Speed: 36.36 +Average Real Time Speed: 31.25 === Per-chunk Stats === -Average chunk size (tokens): 1018.75 +Average chunk size (tokens): 1715.00 Min chunk size (tokens): 150 -Max chunk size (tokens): 3000 -Average processing time (s): 8.75 -Average output length (s): 318.71 +Max chunk size (tokens): 5000 +Average processing time (s): 15.39 +Average output length (s): 529.64 === Performance Ranges === -Processing rate range (tokens/s): 107.14 - 145.63 -RTF range: 0.02x - 0.03x -Real Time Speed range: 33.33x - 50.00x +Processing rate range (tokens/s): 80.65 - 125.10 +RTF range: 0.03x - 0.04x +Real Time Speed range: 25.00x - 33.33x diff --git a/examples/assorted_checks/benchmarks/output_plots/gpu_processing_time_rtf.png b/examples/assorted_checks/benchmarks/output_plots/gpu_processing_time_rtf.png index 476122e..62c6864 100644 Binary files a/examples/assorted_checks/benchmarks/output_plots/gpu_processing_time_rtf.png and b/examples/assorted_checks/benchmarks/output_plots/gpu_processing_time_rtf.png differ diff --git a/examples/assorted_checks/benchmarks/output_plots/gpu_realtime_factor_rtf.png b/examples/assorted_checks/benchmarks/output_plots/gpu_realtime_factor_rtf.png index 2819b09..1c5d7b7 100644 Binary files a/examples/assorted_checks/benchmarks/output_plots/gpu_realtime_factor_rtf.png and b/examples/assorted_checks/benchmarks/output_plots/gpu_realtime_factor_rtf.png differ diff --git a/examples/assorted_checks/benchmarks/output_plots/gpu_system_usage_rtf.png b/examples/assorted_checks/benchmarks/output_plots/gpu_system_usage_rtf.png index 4f8a1b5..942b3a8 100644 Binary files a/examples/assorted_checks/benchmarks/output_plots/gpu_system_usage_rtf.png and b/examples/assorted_checks/benchmarks/output_plots/gpu_system_usage_rtf.png differ