First Round Async. Much work left to do, dont trust results of hash categorization presently.

2025-10-16 16:43:56 -04:00
parent fa0c18ee02
commit 6f2355fea9
21 changed files with 903 additions and 1647 deletions
@@ -0,0 +1,108 @@
+import asyncio
+import logging
+from typing import Any, Callable
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncTaskQueue:
+    def __init__(self, worker_count: int = 3):
+        self.queue = asyncio.Queue()
+        self.worker_count = worker_count
+        self.workers = []
+        self._stop_event = asyncio.Event()
+
+    async def start_workers(self):
+        """Start the worker pool."""
+        logger.debug(f"Starting {self.worker_count} workers...")
+        for i in range(self.worker_count):
+            worker = asyncio.create_task(self.worker_loop(f"Worker-{i+1}"))
+            self.workers.append(worker)
+        logger.debug("All workers started.")
+
+    async def stop_workers(self):
+        """Stop the worker pool and wait for all tasks to complete."""
+        logger.debug("Stopping workers...")
+        self._stop_event.set()  # Signal workers to stop
+        await self.queue.join()  # Wait for all tasks to be processed
+        for worker in self.workers:
+            worker.cancel()
+        await asyncio.gather(*self.workers, return_exceptions=True)
+        logger.debug("All workers stopped.")
+
+
+    async def worker_loop(self, name: str):
+        """Worker loop: Process tasks from the queue."""
+        logger.debug(f"{name} started.")
+        while not self._stop_event.is_set():
+            task = None
+            try:
+                task = await self.queue.get()
+                logger.info(f"{name} processing: {task['name']}")
+                await task['func'](*task['args'], **task['kwargs'])            
+            except asyncio.CancelledError:
+                logger.debug(f"{name} received cancellation.")
+                break
+            except Exception as e:
+                logger.error(f"Error in {name}: {e}", exc_info=True)
+            finally:
+                if task is not None:
+                    self.queue.task_done()
+        logger.debug(f"{name} exited.")
+
+
+
+    async def enqueue(self, name: str, func: Callable, *args: Any, **kwargs: Any):
+        """Add a task to the queue."""
+        logger.debug(f"Enqueuing task: {name}")
+        await self.queue.put({'name': name, 'func': func, 'args': args, 'kwargs': kwargs})
+
+
+
+async def run_sync_task_in_thread(func: Callable, *args: Any, **kwargs: Any):
+    """Run a synchronous function in a separate thread.
+
+    Args:
+        func: The synchronous function to run.
+        *args: Arguments to pass to the function.
+    """
+    await asyncio.to_thread(func, *args, **kwargs)
+    
+    """
+from asyncTaskQueue import AsyncTaskQueue, run_sync_task_in_thread
+import asyncio
+
+# Example async task
+async def your_async_function(name: str, duration: int):
+    print(f"{name} started, will sleep for {duration} seconds")
+    await asyncio.sleep(duration)
+    print(f"{name} finished")
+
+# Example sync task
+def your_sync_function(name: str, duration: int):
+    print(f"{name} started, will sleep for {duration} seconds")
+    import time
+    time.sleep(duration)
+    print(f"{name} finished")
+
+async def main():
+    queue = AsyncTaskQueue(worker_count=2)
+    await queue.start_workers()
+
+    # Enqueue async tasks directly
+    await queue.enqueue("AsyncTask1", your_async_function, "AsyncTask1", 2)
+    await queue.enqueue("AsyncTask2", your_async_function, "AsyncTask2", 1)
+
+    # Enqueue sync tasks using the wrapper
+    await queue.enqueue("SyncTask1", run_sync_task_in_thread, your_sync_function, "SyncTask1", 2)
+
+    # Let tasks run for a while
+    await asyncio.sleep(5)
+
+    # Stop workers
+    await queue.stop_workers()
+
+asyncio.run(main())
+
+
+"""