RC 1.1.1

Development/WIP/Server/datastore.py

@@ -0,0 +1,170 @@
+"""
+datastore.py
+
+Utility module for saving/loading objects to/from JSON and inserting/updating/reading them in SQLite.
+
+Supports any class with:
+- a `to_dict()` method
+- a constructor accepting `**kwargs`
+- optionally, a `from_dict()` method
+
+Author: Brandon Wickline, James Brotosky
+License: GNU Affero General Public License v3.0
+"""
+
+import json
+import sqlite3
+import logging
+from typing import Type, List, TypeVar
+
+# Setup logger
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+# Generic type variable for typing
+T = TypeVar('T')
+
+# -------------------------------------------------------------------
+# JSON SAVE
+# -------------------------------------------------------------------
+def save_to_json(obj_list: List[T], filepath: str) -> None:
+    """
+    Save a list of objects to a JSON file using their `to_dict()` method.
+
+    Args:
+        obj_list (List[T]): List of objects to serialize.
+        filepath (str): Path to the output JSON file.
+
+    Example:
+        save_to_json(policies, "policies.json")
+    """
+    try:
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump([obj.to_dict() for obj in obj_list], f, ensure_ascii=False, indent=4) # pyright: ignore[reportAttributeAccessIssue]
+        logger.info(f"Saved {len(obj_list)} objects to {filepath}")
+    except Exception as e:
+        logger.error(f"Failed to save to {filepath}: {e}")
+
+# -------------------------------------------------------------------
+# JSON LOAD
+# -------------------------------------------------------------------
+def load_from_json(cls: Type[T], filepath: str) -> List[T]:
+    """
+    Load a list of objects from a JSON file and instantiate them using the class constructor
+    or a `from_dict()` method if available.
+
+    Args:
+        cls (Type[T]): Class type to instantiate.
+        filepath (str): Path to the input JSON file.
+
+    Returns:
+        List[T]: List of instantiated objects.
+
+    Example:
+        loaded_agents = load_from_json(Agent, "agents.json")
+    """
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data_list = json.load(f)
+        logger.info(f"Loaded {len(data_list)} records from {filepath}")
+        if hasattr(cls, "from_dict"):
+            return [cls.from_dict(data) for data in data_list] # pyright: ignore[reportAttributeAccessIssue]
+        return [cls(**data) for data in data_list]
+    except Exception as e:
+        logger.error(f"Failed to load from {filepath}: {e}")
+        return []
+
+# -------------------------------------------------------------------
+# SQLITE INSERT OR UPDATE
+# -------------------------------------------------------------------
+def insert_or_update_objects_to_sqlite(obj_list: List[T], table_name: str, db_path: str, primary_key: str) -> None:
+    """
+    Insert or update a list of objects into a SQLite table.
+    Uses `ON CONFLICT(primary_key) DO UPDATE` for upsert behavior.
+
+    Args:
+        obj_list (List[T]): List of objects with `to_dict()` method.
+        table_name (str): Name of the SQLite table.
+        db_path (str): Path to the SQLite database file.
+        primary_key (str): Field name to use as the primary key.
+
+    Example:
+        insert_or_update_objects_to_sqlite(hashes, "hashes", "data_store.db", primary_key="sha256")
+    """
+    if not obj_list:
+        logger.warning("No objects to insert or update into SQLite.")
+        return
+
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        sample_dict = obj_list[0].to_dict() # pyright: ignore[reportAttributeAccessIssue]
+        columns = ', '.join(sample_dict.keys())
+        placeholders = ', '.join(['?'] * len(sample_dict))
+        update_clause = ', '.join([f"{key}=excluded.{key}" for key in sample_dict.keys() if key != primary_key])
+
+        # Create table if it doesn't exist
+        create_stmt = f"""
+        CREATE TABLE IF NOT EXISTS {table_name} (
+            {', '.join([f"{key} TEXT" for key in sample_dict.keys()])},
+            PRIMARY KEY ({primary_key})
+        )
+        """
+        cursor.execute(create_stmt)
+
+        # Insert or update each object
+        for obj in obj_list:
+            values = tuple(str(v) if v is not None else "" for v in obj.to_dict().values()) # pyright: ignore[reportAttributeAccessIssue]
+            insert_stmt = f"""
+            INSERT INTO {table_name} ({columns}) VALUES ({placeholders})
+            ON CONFLICT({primary_key}) DO UPDATE SET {update_clause}
+            """
+            cursor.execute(insert_stmt, values)
+
+        conn.commit()
+        conn.close()
+        logger.info(f"Inserted or updated {len(obj_list)} records into {table_name} table in {db_path}")
+    except Exception as e:
+        logger.error(f"Failed to insert or update into SQLite: {e}")
+
+# -------------------------------------------------------------------
+# SQLITE READ
+# -------------------------------------------------------------------
+def read_objects_from_sqlite(cls: Type[T], table_name: str, db_path: str) -> List[T]:
+    """
+    Generic function to read rows from a SQLite table and convert them into class instances.
+
+    Args:
+        cls (Type[T]): The class to instantiate (e.g., Agent, Policy, Hash).
+        table_name (str): The name of the table to query.
+        db_path (str): Path to the SQLite database file.
+
+    Returns:
+        List[T]: List of class instances.
+
+    Example:
+        agents = read_objects_from_sqlite(Agent, "agents", "data_store.db")
+    """
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute(f"SELECT * FROM {table_name}")
+        rows = cursor.fetchall()
+        columns = [desc[0] for desc in cursor.description]
+
+        objects = []
+        for row in rows:
+            data = dict(zip(columns, row))
+            if hasattr(cls, "from_dict"):
+                obj = cls.from_dict(data) # pyright: ignore[reportAttributeAccessIssue]
+            else:
+                obj = cls(**data)
+            objects.append(obj)
+
+        conn.close()
+        logger.info(f"Read {len(objects)} records from {table_name} table in {db_path}")
+        return objects
+    except Exception as e:
+        logger.error(f"Failed to read from {table_name} in {db_path}: {e}")
+        return []

Development/WIP/Server/localApproval.py

@@ -0,0 +1,291 @@
+#    Copyright (C) 2025 James Brotosky, Brandon Wickline
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import datetime
+import logging
+import os
+import re
+import time
+
+import dotenv
+import numpy as np
+import pandas as pd
+
+from models.agent import Agent
+from services.agenthandler import findAllAgents, moveAgentToRelatedPolicy, selectAgents
+from services.API import AirlockAPIWrapper
+from utils.configmanager import get_protected_json, load_env, load_env_json
+from utils.setup import get_base_directory
+from utils.utils import colorText, get_sanitized_input
+
+logger = logging.getLogger(__name__)
+
+dotenv.load_dotenv()
+
+
+def getLocalApprovals(api: AirlockAPIWrapper):
+    base_dir = get_base_directory
+    result = api.otp_find_awaiting()
+    local_approval = pd.DataFrame(result["response"]["otpusage"])
+    if os.path.exists(f"{base_dir}\\cache\\newest_local_approval.parquet"):
+        previous_run = pd.read_parquet(f"{base_dir}\\cache\\newest_local_approval.parquet")
+        previous_run.to_parquet(
+            f"{base_dir}\\cache\\last_local_approval.parquet", index=False
+        )
+        os.remove(f"{base_dir}\\cache\\newest_local_approval.parquet")
+
+    # Only keep rows presumably created by the generate local approval function
+    local_approval = local_approval[
+        local_approval["purpose"].str.startswith("🎫 Local Approval 🎫")
+    ]
+
+    local_approval["batchid"] = local_approval["purpose"].apply(
+        lambda x: (match := re.search(r"batch:(\S+)", str(x))) and match.group(1)
+    )
+
+    if not local_approval.empty:
+        local_approval.to_parquet(
+            f"{base_dir}\\cache\\newest_local_approval.parquet", index=False
+        )
+
+    return local_approval
+
+
+def scheduleAddingLAHashes(api: AirlockAPIWrapper):
+
+    policy_relationship_map = get_protected_json("POLICY_MAP_ENF_AUD","{}")
+    bad_publisher_list = load_env_json("BAD_PUBLISHER", "[]")
+    pups = load_env_json("PUPS", "[]")
+    threat_tolerance_constant = load_env("VT_THREAT_TOLERANCE", cast_type = int)
+
+    try:
+        register_function("add_hash", returnFromLocalApproval)
+        register_function("move_device", moveAgentToRelatedPolicy)
+    except Exception as e:
+        logger.warning(f"Failed to register functions: {e}")
+        return
+
+    try:
+        approvals_df = getNewLocalApprovals(api)
+        if approvals_df.empty:
+            logger.debug("No new local approvals found. Nothing to schedule.")
+            return
+        batches = approvals_df.groupby("batchid")
+    except Exception as e:
+        logger.warning(f"Failed to retrieve or group local approvals: {e}")
+        return
+
+    for batchid, batch_df in batches:
+        try:
+            duration_minutes = int(batch_df["duration"].iloc[0])
+            start_time = datetime.datetime.now()
+            run_time = start_time + datetime.timedelta(minutes=duration_minutes)
+            early_time = start_time + datetime.timedelta(minutes=np.floor(duration_minutes * 0.95))
+
+            early_timestamp = early_time.timestamp()
+            run_timestamp = run_time.timestamp()
+
+            # Schedule add_hash job
+            try:
+                run_once_job(
+                    f"add_hash_{batchid}",
+                    "add_hash",
+                    early_timestamp,
+                    [
+                        api,
+                        batch_df,
+                        policy_relationship_map,
+                        bad_publisher_list,
+                        pups,
+                        threat_tolerance_constant,
+                    ],
+                    None,
+                )
+                logger.debug(f"Scheduled add_hash for batch {batchid} at {early_time}")
+            except Exception:
+                logger.debug("Failed to schedule add_hash for batch {batchid}: {e}")
+
+            # Schedule move_device jobs
+            devices = batch_df["agentid"].drop_duplicates().tolist()
+            agents = []
+
+            for device in devices:
+                rows = api.agent_find_by_hostname(device).iterrows()
+                agents += [Agent(**row["data"]) for _, row in rows]
+
+            for agent in agents:
+                try:
+                    run_once_job(
+                        f"move_device_{agent.hostame}_{batchid}",
+                        "move_device",
+                        run_timestamp,
+                        [api, agent, policy_relationship_map],
+                        "enforcement",
+                    )
+
+                    print(
+                        f"Scheduled move_device for device {agent.hostname} in batch {batchid} at {run_time}"
+                    )
+                except Exception as e:
+                    print(
+                        f"Failed to schedule move_device for device {agent.hostname} in batch {batchid}: {e}"
+                    )
+
+        except Exception as e:
+            logger.warning(f"Failed to process batch {batchid}: {e}")
+
+
+def returnFromLocalApproval(api, device_df, policy_relationship_map, bad_publisher_list, pups, threat_tolerance_constant
+):
+    """
+      # Get unique policy names from device list
+      policies_in_devicelist = sorted(device_df['policy_name'].unique().tolist())
+
+      # Create inverse map to go from Audit to Enforcement
+      inverse_map = {v: k for k, v in policy_relationship_map.items()}
+
+      # Fetch all policies
+      all_policies = [Policy(row['groupid'], row['hidden'], row['name'], row['parent']) for _, row in api.policy_find_all().iterrows()]
+
+      # Define policy types
+      policy_types = [1, 2, 6, 7]
+
+    #TODO finish logic for adding hashes
+    """
+    working_dir = load_env("WORKING_DIR")
+    policy_relationship_map = get_protected_json("POLICY_MAP_ENF_AUD","{}")
+    bad_publisher_list = load_env_json("BAD_PUBLISHER", "[]")
+    pups = load_env_json("PUPS", "[]")
+    threat_tolerance_constant = load_env("VT_THREAT_TOLERANCE")
+    print(f"{working_dir}, {policy_relationship_map}, {bad_publisher_list}, {pups}, {threat_tolerance_constant}")
+
+def moveToLocalApproval(api: AirlockAPIWrapper):
+    possible_durations = [15, 60, 360, 1440, 10080]
+    duration_selected = None
+
+    print(colorText("Please select a duration:", "white"))
+    for i, option in enumerate(possible_durations, start=1):
+        print(f"{i}. {option}")
+
+    try:
+
+        choice = int(get_sanitized_input("Enter the number of your choice:"))
+        if 1 <= choice <= len(possible_durations):
+            duration_selected = possible_durations[choice - 1]
+            print(colorText(f"You selected: {duration_selected}", "yellow"))
+            logger.debug(f"You selected: {duration_selected}")
+        else:
+            print(colorText("❌ Invalid choice.", "red"))
+            logger.debug("Invalid Input")
+            return
+    except ValueError:
+        print(colorText("❌ Invalid input. Please enter a number.", "red"))
+        logger.debug("Invalid Input")
+        return
+
+    agents = selectAgents(api)
+    batch = int(time.time())
+
+    if not agents:
+        print(colorText("❌ No agents found or error retrieving agents.", "red"))
+        logger.debug("No agents found or error retrieving agents")
+        return
+
+    for agent in agents:
+        try:
+            addLocalApproval(api, batch, duration_selected, agent.agentid)
+            moveAgentToRelatedPolicy(api, agent, "audit")
+        except Exception as e:
+            print(colorText(f"❌ Error processing agent {agent.hostname}: {e}", "red"))
+
+
+def addLocalApproval(api: AirlockAPIWrapper, batchid, duration_selected, agentid):
+    
+    purpose = f"🎫 Local Approval 🎫 - {duration_selected} mins - batch:{batchid} Client:{agentid}"
+    api.otp_generate(agentid, duration_selected, purpose)
+ 
+
+
+def monitorAuditStatus(api: AirlockAPIWrapper):
+    current_agents = findAllAgents(api)
+    last_agents = []
+    if not last_agents:
+        last_agents = current_agents
+    policy_relationship_map = get_protected_json("POLICY_MAP_ENF_AUD","{}")
+
+    # Reverse map for audit → enforcement
+    reverse_policy_map = {v: k for k, v in policy_relationship_map.items()}
+    known_transitions = set(policy_relationship_map.items()) | set(reverse_policy_map.items())
+
+    # Index last_agents by hostname for quick lookup
+    last_agent_map = {agent.hostname: agent for agent in last_agents}
+
+    # Result buckets
+    newly_added = []
+    same_policy = []
+    moved_to_audit = []
+    moved_to_enforcement = []
+    unusual_move = []
+
+    for current in current_agents:
+        previous = last_agent_map.get(current.hostname)
+
+        if not previous:
+            newly_added.append(current)
+            continue
+
+        if current.groupid == previous.groupid:
+            same_policy.append(current)
+        elif (previous.groupid, current.groupid) in known_transitions:
+            moved_to_audit.append(current)
+        elif (current.groupid, previous.groupid) in known_transitions:
+            moved_to_enforcement.append(current)
+        else:
+            unusual_move.append(current)
+
+    # Return all five DataFrames
+    return newly_added, same_policy, moved_to_audit, moved_to_enforcement, unusual_move
+
+
+def getNewLocalApprovals(api: AirlockAPIWrapper):
+    
+    working_dir = load_env("WORKING_DIR")                   
+    current_la = getLocalApprovals(api)
+
+    # Load old approval list
+    old_la_path = f"{working_dir}\\Scheduling\\last_local_approval.parquet"
+    if os.path.exists(old_la_path):
+        old_la = pd.read_parquet(old_la_path)
+    else:
+        old_la = pd.DataFrame(columns=current_la.columns)
+
+    # Create composite keys
+    current_la["key"] = current_la["clientid"].astype(str) + "_" + current_la["granted"].astype(str)
+    old_la["key"] = old_la["clientid"].astype(str) + "_" + old_la["granted"].astype(str)
+
+    # Find new entries
+    new_entries = current_la[~current_la["key"].isin(old_la["key"])]
+
+    # Convert 'granted' to datetime and filter by last 10 minutes
+    new_entries["granted"] = pd.to_datetime(new_entries["granted"], utc=True, errors="coerce")
+    ten_minutes_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=10)
+    recent_entries = new_entries[new_entries["granted"] > ten_minutes_ago]
+
+    # Save current approvals for next run
+    current_la.drop(columns=["key"], inplace=True)
+    current_la.to_parquet(old_la_path, index=False)
+
+    return recent_entries

Development/WIP/Server/scheduler.py

@@ -0,0 +1,352 @@
+#    Copyright (C) 2025 James Brotosky, Brandon Wickline
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import json
+import logging
+import os
+import time
+from typing import Any, Callable, Dict, List
+
+import schedule
+
+logger = logging.getLogger(__name__)
+
+
+# TODO - move this File where all jobs are persisted
+JOBS_FILE = f"{os.getenv('WORKING_DIR')}\\scheduling\\jobs.json"
+
+# Ensure directory exists
+os.makedirs(os.path.dirname(JOBS_FILE), exist_ok=True)
+
+# Registry of functions that can be scheduled
+FUNCTION_MAP: Dict[str, Callable] = {}
+
+# -------------------------------
+# Function Registration
+# -------------------------------
+
+
+def register_function(name: str, func: Callable):
+    """
+    Register a function so it can be called by name later.
+    Example:
+        register_function("say_hello", say_hello)
+    """
+    FUNCTION_MAP[name] = func
+
+
+# -------------------------------
+# Persistence Helpers
+# -------------------------------
+
+
+def load_jobs() -> List[Dict[str, Any]]:
+    """Load jobs from the JSON file, or return [] if none exist."""
+    if not os.path.exists(JOBS_FILE):
+        return []
+    with open(JOBS_FILE, "r") as f:
+        return json.load(f)
+
+
+def _atomic_save(path: str, data: Any):
+    """Write JSON atomically to avoid partial writes."""
+    tmp = f"{path}.tmp"
+    with open(tmp, "w") as f:
+        json.dump(data, f, indent=4)
+    os.replace(tmp, path)
+
+
+def save_jobs(jobs: List[Dict[str, Any]]):
+    """Save jobs to the JSON file (overwrite)."""
+    _atomic_save(JOBS_FILE, jobs)
+
+
+# -------------------------------
+# Uniqueness Helpers
+# -------------------------------
+
+
+def job_in_store(job_id: str) -> bool:
+    """Check if a job id exists in the persisted JSON file."""
+    return any(j.get("id") == job_id for j in load_jobs())
+
+
+def job_in_scheduler(job_id: str) -> bool:
+    """
+    Check if a job with this tag exists in the in-memory scheduler.
+    Uses schedule.get_jobs(tag=...) if available, otherwise scans tags.
+    """
+    try:
+        jobs = schedule.get_jobs(tag=job_id)  # schedule >= 1.2.0
+        return len(jobs) > 0
+    except TypeError:
+        # Fallback for older versions
+        return any(job_id in getattr(j, "tags", set()) for j in schedule.jobs)
+
+
+def ensure_unique(job_id: str, on_conflict: str = "skip") -> bool:
+    """
+    Ensure the job_id is unique across persistence and in-memory schedule.
+    on_conflict:
+        - "error": raise ValueError if exists.
+        - "skip" : print and return False.
+        - "replace": remove existing (in-memory + JSON), then continue.
+    """
+    exists = job_in_store(job_id) or job_in_scheduler(job_id)
+    if not exists:
+        return True
+    if on_conflict == "error":
+        raise ValueError(f"Job id '{job_id}' already exists.")
+    elif on_conflict == "skip":
+        logger.info(f"Job '{job_id}' already exists. Skipping creation.")
+        return False
+    elif on_conflict == "replace":
+        # Clear from scheduler
+        schedule.clear(job_id)
+        # Remove from persistence
+        jobs = [j for j in load_jobs() if j.get("id") != job_id]
+        save_jobs(jobs)
+        return True
+    else:
+        raise ValueError(f"Unsupported on_conflict policy: {on_conflict}")
+
+
+# -------------------------------
+# Internal scheduling (no persistence)
+# -------------------------------
+
+
+def _schedule_once(job_id: str, func_name: str, run_at_timestamp: float, args=None, kwargs=None):
+    args = args or []
+    kwargs = kwargs or {}
+
+    def job_wrapper():
+        """Executes the job once, then removes it."""
+        if func_name not in FUNCTION_MAP:
+            logger.error(f"Function '{func_name}' is not registered.")
+            return
+        FUNCTION_MAP[func_name](*args, **kwargs)
+        # Remove from persistence
+        jobs = load_jobs()
+        jobs = [j for j in jobs if j["id"] != job_id]
+        save_jobs(jobs)
+        # Clear from in-memory schedule
+        schedule.clear(job_id)
+
+    delay_seconds = run_at_timestamp - time.time()
+    if delay_seconds <= 0:
+        logger.info(f"Job {job_id} scheduled in the past. Skipping.")
+        return
+    # Schedule via schedule library
+    schedule.every(int(delay_seconds)).seconds.do(job_wrapper).tag(job_id)
+
+
+def _schedule_recurring(
+    job_id: str, func_name: str, interval: int, unit: str, args=None, kwargs=None
+):
+    args = args or []
+    kwargs = kwargs or {}
+
+    def job_wrapper():
+        if func_name not in FUNCTION_MAP:
+            logger.error(f"Function '{func_name}' is not registered.")
+            return
+        FUNCTION_MAP[func_name](*args, **kwargs)
+
+    if unit == "seconds":
+        schedule.every(interval).seconds.do(job_wrapper).tag(job_id)
+    elif unit == "minutes":
+        schedule.every(interval).minutes.do(job_wrapper).tag(job_id)
+    elif unit == "hours":
+        schedule.every(interval).hours.do(job_wrapper).tag(job_id)
+    elif unit == "days":
+        schedule.every(interval).days.do(job_wrapper).tag(job_id)
+    else:
+        raise ValueError(f"Unsupported unit: {unit}")
+
+
+# -------------------------------
+# Public APIs (with uniqueness + persistence)
+# -------------------------------
+
+
+def run_once_job(
+    job_id: str,
+    func_name: str,
+    run_at_timestamp: float,
+    args=None,
+    kwargs=None,
+    *,
+    replace: bool = False,
+    persist: bool = True,
+):
+    """
+    Schedule a job to run once at a specific timestamp.
+    replace: if True, replace existing job with same id; otherwise print and skip.
+    persist: if False, do not write to JSON (used by reload_jobs()).
+    """
+    if persist:
+        policy = "replace" if replace else "skip"
+        if not ensure_unique(job_id, on_conflict=policy):
+            return
+    elif job_in_scheduler(job_id):
+        schedule.clear(job_id)
+    _schedule_once(job_id, func_name, run_at_timestamp, args, kwargs)
+    if persist:
+        jobs = [j for j in load_jobs() if j["id"] != job_id]
+        jobs.append(
+            {
+                "id": job_id,
+                "type": "once",
+                "run_at": run_at_timestamp,
+                "function": func_name,
+                "args": args or [],
+                "kwargs": kwargs or {},
+            }
+        )
+        save_jobs(jobs)
+
+
+def recurring_job(
+    job_id: str,
+    func_name: str,
+    interval: int,
+    unit: str,
+    args=None,
+    kwargs=None,
+    *,
+    replace: bool = False,
+    persist: bool = True,
+):
+    """
+    Schedule a recurring job.
+    replace: if True, replace existing job with same id; otherwise print and skip.
+    persist: if False, do not write to JSON (used by reload_jobs()).
+    """
+    if persist:
+        policy = "replace" if replace else "skip"
+        if not ensure_unique(job_id, on_conflict=policy):
+            return
+    elif job_in_scheduler(job_id):
+        schedule.clear(job_id)
+    _schedule_recurring(job_id, func_name, interval, unit, args, kwargs)
+    if persist:
+        jobs = [j for j in load_jobs() if j["id"] != job_id]
+        jobs.append(
+            {
+                "id": job_id,
+                "type": "recurring",
+                "interval": interval,
+                "unit": unit,
+                "function": func_name,
+                "args": args or [],
+                "kwargs": kwargs or {},
+            }
+        )
+        save_jobs(jobs)
+
+
+def find_and_prioritize_jobs_by_pid(pid_substring: str, new_delay_seconds: float = 1.0):
+    """
+    Find all jobs whose ID contains the given PID substring and reschedule them to run sooner.
+    """
+    jobs = load_jobs()
+    matched_jobs = [job for job in jobs if pid_substring in job.get("id", "")]
+
+    if not matched_jobs:
+        logger.info(f"No jobs found containing PID substring '{pid_substring}'.")
+        return
+
+    logger.info(f"Found {len(matched_jobs)} job(s) containing '{pid_substring}':")
+
+    for job in matched_jobs:
+        job_id = job["id"]
+        logger.debug(f"  - Prioritizing job: {job_id}")
+
+        # Clear existing job from scheduler
+        schedule.clear(job_id)
+
+        # Reschedule based on job type
+        if job["type"] == "once":
+            run_once_job(
+                job_id,
+                job["function"],
+                time.time() + new_delay_seconds,
+                job.get("args"),
+                job.get("kwargs"),
+                replace=True,
+                persist=True,
+            )
+        elif job["type"] == "recurring":
+            recurring_job(
+                job_id,
+                job["function"],
+                job["interval"],
+                job["unit"],
+                job.get("args"),
+                job.get("kwargs"),
+                replace=True,
+                persist=True,
+            )
+        else:
+            logger.warning(f"Unknown job type for job '{job_id}'")
+
+
+# -------------------------------
+# Reload Saved Jobs
+# -------------------------------
+
+
+def reload_jobs():
+    """Reload jobs from JSON and reschedule them (no re-persist)."""
+    jobs = load_jobs()
+    for job in jobs:
+        if job["type"] == "once":
+            if job["run_at"] > time.time():
+                run_once_job(
+                    job["id"],
+                    job["function"],
+                    job["run_at"],
+                    job.get("args"),
+                    job.get("kwargs"),
+                    persist=False,
+                )
+        elif job["type"] == "recurring":
+            recurring_job(
+                job["id"],
+                job["function"],
+                job["interval"],
+                job["unit"],
+                job.get("args"),
+                job.get("kwargs"),
+                persist=False,
+            )
+
+
+# -------------------------------
+# Scheduler Loop
+# -------------------------------
+
+
+def start_scheduler():
+    """
+    Start the scheduler loop (blocking).
+    Call this once in main to begin.
+    """
+    try:
+        while True:
+            schedule.run_pending()
+            time.sleep(0.5)
+    except KeyboardInterrupt:
+        logger.critical("Scheduler stopped.")

Development/WIP/Server/scheduler_async.py

@@ -0,0 +1,177 @@
+import asyncio
+import json
+import logging
+import os
+from typing import Any, Callable, Dict, List
+
+logger = logging.getLogger(__name__)
+
+# Registry of functions that can be scheduled
+FUNCTION_MAP: Dict[str, Callable] = {}
+
+# Dictionary to manually track scheduled jobs by ID
+scheduled_jobs: Dict[str, asyncio.TimerHandle] = {}
+
+# Path to the JSON file for job persistence TODO - pin this to the correct place
+JOBS_FILE = os.path.join(os.getcwd(), "jobs.json")
+
+def register_function(name: str, func: Callable):
+    """
+    Register a function so it can be called by name later.
+    Example:
+        register_function("say_hello", say_hello)
+    """
+    FUNCTION_MAP[name] = func
+
+def load_jobs() -> List[Dict[str, Any]]:
+    """
+    Load jobs from the JSON file, or return [] if none exist.
+    """
+    if not os.path.exists(JOBS_FILE):
+        return []
+    with open(JOBS_FILE, "r") as f:
+        return json.load(f)
+
+def save_jobs(jobs: List[Dict[str, Any]]):
+    """
+    Save jobs to the JSON file (overwrite).
+    """
+    with open(JOBS_FILE, "w") as f:
+        json.dump(jobs, f, indent=4)
+
+def cancel_job(job_id: str):
+    """
+    Cancel a scheduled job by ID and remove it from the registry and persistence.
+    """
+    handle = scheduled_jobs.pop(job_id, None)
+    if handle:
+        handle.cancel()
+        logger.info(f"Cancelled job '{job_id}'")
+
+    jobs = [j for j in load_jobs() if j.get("id") != job_id]
+    save_jobs(jobs)
+
+def run_once_job(job_id: str, func_name: str, delay_seconds: float, args=None, kwargs=None, persist=True):
+    """
+    Schedule a job to run once after a delay (in seconds).
+    """
+    args = args or []
+    kwargs = kwargs or {}
+
+    def job_wrapper():
+        func = FUNCTION_MAP.get(func_name)
+        if func is None:
+            logger.error(f"Function '{func_name}' is not registered.")
+            return
+        func(*args, **kwargs)
+        cancel_job(job_id)
+
+    loop = asyncio.get_event_loop()
+    handle = loop.call_later(delay_seconds, job_wrapper)
+    scheduled_jobs[job_id] = handle
+
+    if persist:
+        jobs = [j for j in load_jobs() if j.get("id") != job_id]
+        jobs.append({
+            "id": job_id,
+            "type": "once",
+            "delay": delay_seconds,
+            "function": func_name,
+            "args": args,
+            "kwargs": kwargs
+        })
+        save_jobs(jobs)
+        logger.info(f"Scheduled one-time job '{job_id}' to run in {delay_seconds} seconds.")
+
+def recurring_job(job_id: str, func_name: str, interval: float, args=None, kwargs=None, persist=True):
+    """
+    Schedule a recurring job.
+    """
+    args = args or []
+    kwargs = kwargs or {}
+
+    def job_wrapper():
+        func = FUNCTION_MAP.get(func_name)
+        if func is None:
+            logger.error(f"Function '{func_name}' is not registered.")
+            return
+        func(*args, **kwargs)
+        # Reschedule the job
+        handle = asyncio.get_event_loop().call_later(interval, job_wrapper)
+        scheduled_jobs[job_id] = handle
+
+    cancel_job(job_id)
+    handle = asyncio.get_event_loop().call_later(interval, job_wrapper)
+    scheduled_jobs[job_id] = handle
+
+    if persist:
+        jobs = [j for j in load_jobs() if j.get("id") != job_id]
+        jobs.append({
+            "id": job_id,
+            "type": "recurring",
+            "interval": interval,
+            "function": func_name,
+            "args": args,
+            "kwargs": kwargs
+        })
+        save_jobs(jobs)
+        logger.info(f"Scheduled recurring job '{job_id}' every {interval} seconds.")
+
+def reload_jobs():
+    """
+    Reload jobs from JSON and reschedule them.
+    """
+    jobs = load_jobs()
+    for job in jobs:
+        if job["type"] == "once":
+            run_once_job(
+                job["id"],
+                job["function"],
+                job["delay"],
+                job.get("args"),
+                job.get("kwargs"),
+                persist=False
+            )
+        elif job["type"] == "recurring":
+            recurring_job(
+                job["id"],
+                job["function"],
+                job["interval"],
+                job.get("args"),
+                job.get("kwargs"),
+                persist=False
+            )
+
+async def start_scheduler():
+    """
+    Start the asynchronous scheduler loop.
+
+    This function is a placeholder to keep the event loop alive.
+    Jobs are scheduled using asyncio.call_later and do not require polling.
+    """
+    try:
+        await asyncio.Event().wait()
+    except asyncio.CancelledError:
+        logger.critical("Scheduler stopped.")
+
+    """
+    Start the asynchronous scheduler loop.
+
+    This function is a placeholder for compatibility. Since we use asyncio.call_later,
+    jobs are scheduled directly on the event loop and no polling is required.
+
+    Usage:
+        # In an async app (e.g., Textual)
+        asyncio.create_task(start_scheduler())
+
+        # Or in a standalone script
+        async def main():
+            await start_scheduler()
+
+        asyncio.run(main())
+    """
+    try:
+        while True:
+            await asyncio.sleep(3600)  # Sleep indefinitely; jobs run via call_later
+    except asyncio.CancelledError:
+        logger.critical("Scheduler stopped.")

Development/WIP/Server/schema-v1.sql

@@ -0,0 +1,95 @@
+
+-- Current state of agents
+CREATE TABLE IF NOT EXISTS agents (
+    agentid TEXT PRIMARY KEY,
+    clientversion TEXT,
+    domain TEXT,
+    freespace INTEGER,
+    groupid TEXT,
+    hostname TEXT,
+    ip TEXT,
+    localip TEXT,
+    lastcheckin TEXT,
+    os TEXT,
+    policyversion TEXT,
+    status INTEGER,
+    username TEXT,
+    groupname TEXT,
+    status_text TEXT,
+    firstseen TEXT
+);
+
+-- History of clientversion changes
+CREATE TABLE IF NOT EXISTS agent_clientversion_history (
+    agentid TEXT,
+    clientversion TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (agentid, clientversion, timestamp)
+);
+
+-- History of IP changes
+CREATE TABLE IF NOT EXISTS agent_ip_history (
+    agentid TEXT,
+    ip TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (agentid, ip, timestamp)
+);
+
+-- History of local IP changes
+CREATE TABLE IF NOT EXISTS agent_localip_history (
+    agentid TEXT,
+    localip TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (agentid, localip, timestamp)
+);
+
+-- History of policyversion changes
+CREATE TABLE IF NOT EXISTS agent_policyversion_history (
+    agentid TEXT,
+    policyversion TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (agentid, policyversion, timestamp)
+);
+
+
+-- Current state of policies
+CREATE TABLE IF NOT EXISTS policies (
+    groupid TEXT PRIMARY KEY,
+    hidden BOOLEAN,
+    name TEXT,
+    parent TEXT,
+    firstseen TEXT
+);
+
+-- History of policy name changes
+CREATE TABLE IF NOT EXISTS policy_name_history (
+    groupid TEXT,
+    name TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (groupid, name, timestamp)
+);
+
+
+-- Current state of allowlists
+CREATE TABLE IF NOT EXISTS allowlists (
+    applicationid TEXT PRIMARY KEY,
+    name TEXT,
+    version TEXT,
+    firstseen TEXT
+);
+
+-- History of allowlist name changes
+CREATE TABLE IF NOT EXISTS allowlist_name_history (
+    applicationid TEXT,
+    name TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (applicationid, name, timestamp)
+);
+
+-- History of allowlist version changes
+CREATE TABLE IF NOT EXISTS allowlist_version_history (
+    applicationid TEXT,
+    version TEXT,
+    timestamp TEXT,
+    PRIMARY KEY (applicationid, version, timestamp)
+);

Development/WIP/Server/sqlhandler.py

@@ -0,0 +1,232 @@
+import sqlite3
+import logging
+from models.agent import Agent
+from models.policy import Policy, Allowlist
+from datetime import datetime
+from typing import List
+logger = logging.getLogger(__name__)
+
+
+# ------------------ Database Initialization ------------------
+
+def initialize_db(db_path: str, schema_path: str, schema_version: str = "1.0"):
+    try:
+        with open(schema_path, 'r') as f:
+            schema_sql = f.read()
+
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.executescript(schema_sql)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS schema_version (
+                version TEXT,
+                applied_on TEXT
+            )
+        """)
+        cursor.execute("""
+            INSERT INTO schema_version (version, applied_on)
+            VALUES (?, ?)
+        """, (schema_version, datetime.now().isoformat()))
+
+        conn.commit()
+        conn.close()
+        logger.info(f"Tracking database initialized at {db_path} with schema version {schema_version}")
+    except Exception as e:
+        logger.error(f"Error initializing database: {e}")
+
+# ------------------ Change Tracking ------------------
+
+def compare_and_track_changes(
+    conn,
+    current_agents: List[Agent],
+    current_policies: List[Policy],
+    current_allowlists: List[Allowlist]
+):
+    try:
+        cursor = conn.cursor()
+
+        # --- Agents ---
+        cursor.execute("SELECT * FROM agents")
+        agent_rows = cursor.fetchall()
+        agent_columns = [desc[0] for desc in cursor.description]
+        previous_agents = {
+            row[agent_columns.index("agentid")]: Agent(**dict(zip(agent_columns, row)))
+            for row in agent_rows
+        }
+
+        for agent in current_agents:
+            if agent.agentid not in previous_agents:
+                insert_new_agent(conn, agent)
+            else:
+                log_agent_changes(conn, previous_agents[agent.agentid], agent)
+
+        # --- Policies ---
+        cursor.execute("SELECT * FROM policies")
+        policy_rows = cursor.fetchall()
+        policy_columns = [desc[0] for desc in cursor.description]
+        previous_policies = {
+            row[policy_columns.index("groupid")]: Policy(**dict(zip(policy_columns, row)))
+            for row in policy_rows
+        }
+
+        for policy in current_policies:
+            if policy.groupid not in previous_policies:
+                insert_new_policy(conn, policy)
+            else:
+                log_policy_changes(conn, previous_policies[policy.groupid], policy)
+
+        # --- Allowlists ---
+        cursor.execute("SELECT * FROM allowlists")
+        allowlist_rows = cursor.fetchall()
+        allowlist_columns = [desc[0] for desc in cursor.description]
+        previous_allowlists = {
+            row[allowlist_columns.index("applicationid")]: Allowlist(**dict(zip(allowlist_columns, row)))
+            for row in allowlist_rows
+        }
+
+        for allowlist in current_allowlists:
+            if allowlist.applicationid not in previous_allowlists:
+                insert_new_allowlist(conn, allowlist)
+            else:
+                log_allowlist_changes(conn, previous_allowlists[allowlist.applicationid], allowlist)
+
+        conn.commit()
+        logger.info("Change tracking completed successfully.")
+    except Exception as e:
+        logger.error(f"Error during change tracking: {e}")
+
+# ------------------ Insert and Log Functions ------------------
+
+def insert_new_agent(conn, agent: Agent):
+    try:
+        timestamp = datetime.now().isoformat()
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            INSERT INTO agents (
+                agentid, clientversion, domain, freespace, groupid, hostname, ip, localip,
+                lastcheckin, os, policyversion, status, username, groupname, status_text, firstseen
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(agentid) DO NOTHING
+        """, (
+            agent.agentid, agent.clientversion, agent.domain, agent.freespace, agent.groupid,
+            agent.hostname, agent.ip, agent.localip, agent.lastcheckin, agent.os,
+            agent.policyversion, agent.status, agent.username, agent.groupname, agent.status_text,
+            timestamp
+        ))
+
+        for field, value in [
+            ("clientversion", agent.clientversion),
+            ("ip", agent.ip),
+            ("localip", agent.localip),
+            ("policyversion", agent.policyversion),
+            ("hostname", agent.hostname),
+            ("status", agent.status)
+        ]:
+            cursor.execute(f"""
+                INSERT INTO agent_{field}_history (agentid, {field}, timestamp)
+                VALUES (?, ?, ?)
+            """, (agent.agentid, value, timestamp))
+
+        conn.commit()
+        logger.info(f"Inserted new agent: {agent.agentid}")
+    except Exception as e:
+        logger.error(f"Error inserting new agent {agent.agentid}: {e}")
+
+def log_agent_changes(conn, previous: Agent, current: Agent):
+    try:
+        timestamp = datetime.now().isoformat()
+        cursor = conn.cursor()
+
+        for field in ["clientversion", "ip", "localip", "policyversion", "hostname", "status"]:
+            if getattr(previous, field) != getattr(current, field):
+                cursor.execute(f"""
+                    INSERT INTO agent_{field}_history (agentid, {field}, timestamp)
+                    VALUES (?, ?, ?)
+                """, (current.agentid, getattr(current, field), timestamp))
+
+        cursor.execute("""
+            UPDATE agents SET lastcheckin = ? WHERE agentid = ?
+        """, (current.lastcheckin, current.agentid))
+
+        conn.commit()
+        logger.info(f"Logged changes for agent: {current.agentid}")
+    except Exception as e:
+        logger.error(f"Error logger changes for agent {current.agentid}: {e}")
+
+def insert_new_policy(conn, policy: Policy):
+    try:
+        timestamp = datetime.now().isoformat()
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            INSERT INTO policies (groupid, hidden, name, parent, firstseen)
+            VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(groupid) DO NOTHING
+        """, (policy.groupid, policy.hidden, policy.name, policy.parent, timestamp))
+
+        cursor.execute("""
+            INSERT INTO policy_name_history (groupid, name, timestamp)
+            VALUES (?, ?, ?)
+        """, (policy.groupid, policy.name, timestamp))
+
+        conn.commit()
+        logger.info(f"Inserted new policy: {policy.groupid}")
+    except Exception as e:
+        logger.error(f"Error inserting new policy {policy.groupid}: {e}")
+
+def log_policy_changes(conn, previous: Policy, current: Policy):
+    try:
+        timestamp = datetime.now().isoformat()
+        cursor = conn.cursor()
+
+        if previous.name != current.name:
+            cursor.execute("""
+                INSERT INTO policy_name_history (groupid, name, timestamp)
+                VALUES (?, ?, ?)
+            """, (current.groupid, current.name, timestamp))
+
+        conn.commit()
+        logger.info(f"Logged changes for policy: {current.groupid}")
+    except Exception as e:
+        logger.error(f"Error logger changes for policy {current.groupid}: {e}")
+
+def insert_new_allowlist(conn, allowlist: Allowlist):
+    try:
+        timestamp = datetime.now().isoformat()
+        cursor = conn.cursor()
+
+        cursor.execute("""
+            INSERT INTO allowlists (applicationid, name, version, firstseen)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(applicationid) DO NOTHING
+        """, (allowlist.applicationid, allowlist.name, allowlist.version, timestamp))
+
+        for field, value in [("name", allowlist.name), ("version", allowlist.version)]:
+            cursor.execute(f"""
+                INSERT INTO allowlist_{field}_history (applicationid, {field}, timestamp)
+                VALUES (?, ?, ?)
+            """, (allowlist.applicationid, value, timestamp))
+
+        conn.commit()
+        logger.info(f"Inserted new allowlist: {allowlist.applicationid}")
+    except Exception as e:
+        logger.error(f"Error inserting new allowlist {allowlist.applicationid}: {e}")
+
+def log_allowlist_changes(conn, previous: Allowlist, current: Allowlist):
+    try:
+        timestamp = datetime.now().isoformat()
+        cursor = conn.cursor()
+
+        for field in ["name", "version"]:
+            if getattr(previous, field) != getattr(current, field):
+                cursor.execute(f"""
+                    INSERT INTO allowlist_{field}_history (applicationid, {field}, timestamp)
+                    VALUES (?, ?, ?)
+                """, (current.applicationid, getattr(current, field), timestamp))
+
+        conn.commit()
+        logger.info(f"Logged changes for allowlist: {current.applicationid}")
+    except Exception as e:
+        logger.error(f"Error logger changes for allowlist {current.applicationid}: {e}")