Config Refactor: unify config handling

- Consolidated all system/user config logic into configmanager.py - Removed duplicate loaders from setup.py and TUI.py - Eliminated .env redundancy; now only stores WORKING_DIR - Clarified boundaries: system config immutable, user config mutable - Updated TUI to use save_user_config() - Removed all deprecated/legacy config functions and aliases
2025-11-17 12:00:32 -05:00
parent b198362ac8
commit 89654d3a8c
13 changed files with 432 additions and 417 deletions
@@ -10,7 +10,7 @@ from typing import List, Optional
 from models.agent import Agent
 from services.agenthandler import moveAgentToRelatedPolicy, selectAgents
 from services.API import AirlockAPIWrapper
-from utils.configmanager import get_protected_json
+from utils.configmanager import get_system_json
 from utils.utils import colorText, get_sanitized_input

 logger = logging.getLogger(__name__)
@@ -28,7 +28,7 @@ class LocalApprovalRequestor:
            username: Username creating the approvals (for tracking)
        """
        self.api = api
-        self.policy_map = get_protected_json("POLICY_MAP_ENF_AUD", "{}")
+        self.policy_map = get_system_json("POLICY_MAP_ENF_AUD", "{}")
        self.username = (
            username or os.getenv("USERNAME") or os.getenv("USER") or "unknown"
        )
@@ -51,7 +51,7 @@ class LocalApprovalRequestor:
            batch_id = int(time.time())

        purpose = (
-            f"🎫 Local Approval 🎫 - {duration_minutes} mins - "
+            f"ðŸŽ« Local Approval ðŸŽ« - {duration_minutes} mins - "
            f"batch:{batch_id} Client:{agent_id} User:{self.username}"
        )

@@ -103,10 +103,10 @@ class LocalApprovalRequestor:
        success_count = 0
        failure_count = 0

-        print(colorText(f"\n📦 Processing batch {batch_id}...", "cyan"))
-        print(colorText(f"👤 Requested by: {self.username}", "cyan"))
+        print(colorText(f"\nðŸ“¦ Processing batch {batch_id}...", "cyan"))
+        print(colorText(f"ðŸ‘¤ Requested by: {self.username}", "cyan"))
        print(
-            colorText(f"📊 Moving {len(agents)} agent(s) to local approval\n", "cyan")
+            colorText(f"ðŸ“Š Moving {len(agents)} agent(s) to local approval\n", "cyan")
        )

        for agent in agents:
@@ -125,11 +125,11 @@ class LocalApprovalRequestor:
                if not move_success:
                    raise Exception("Failed to move to audit policy")

-                print(colorText(f"✓ {agent.hostname}", "green"))
+                print(colorText(f"âœ“ {agent.hostname}", "green"))
                success_count += 1

            except Exception as e:
-                print(colorText(f"✗ {agent.hostname}: {e}", "red"))
+                print(colorText(f"âœ— {agent.hostname}: {e}", "red"))
                logger.error(f"Error processing agent {agent.hostname}: {e}")
                failure_count += 1

@@ -152,7 +152,7 @@ class LocalApprovalRequestor:
        ]

        # Display duration options
-        print(colorText("\n⏱️  Select Local Approval Duration:", "white"))
+        print(colorText("\nâ±ï¸  Select Local Approval Duration:", "white"))
        print(colorText("=" * 50, "white"))

        for i, (minutes, label) in enumerate(duration_options, start=1):
@@ -166,36 +166,36 @@ class LocalApprovalRequestor:

            if 1 <= choice <= len(duration_options):
                duration_minutes, duration_label = duration_options[choice - 1]
-                print(colorText(f"✓ Selected: {duration_label}", "green"))
+                print(colorText(f"âœ“ Selected: {duration_label}", "green"))
                logger.info(f"User selected duration: {duration_minutes} minutes")
            else:
-                print(colorText("❌ Invalid choice.", "red"))
+                print(colorText("âŒ Invalid choice.", "red"))
                logger.warning("Invalid duration choice")
                return

        except ValueError:
-            print(colorText("❌ Invalid input. Please enter a number.", "red"))
+            print(colorText("âŒ Invalid input. Please enter a number.", "red"))
            logger.warning("Invalid input for duration selection")
            return

        # Select agents
-        print(colorText("\n🎯 Select Agents for Local Approval:", "white"))
+        print(colorText("\nðŸŽ¯ Select Agents for Local Approval:", "white"))
        agents = selectAgents(self.api)

        if not agents:
-            print(colorText("❌ No agents found or error retrieving agents.", "red"))
+            print(colorText("âŒ No agents found or error retrieving agents.", "red"))
            logger.warning("No agents selected or error retrieving agents")
            return

        # Confirm with user
-        print(colorText("\n📋 Summary:", "cyan"))
+        print(colorText("\nðŸ“‹ Summary:", "cyan"))
        print(colorText(f"  Duration: {duration_label}", "white"))
        print(colorText(f"  Agents: {len(agents)}", "white"))

        confirm = get_sanitized_input("\nProceed? (y/n): ").lower()

        if confirm != "y":
-            print(colorText("❌ Operation cancelled.", "yellow"))
+            print(colorText("âŒ Operation cancelled.", "yellow"))
            return

        # Process the batch
@@ -219,24 +219,25 @@ class LocalApprovalRequestor:
            failure_count: Number of failed operations
        """
        print(colorText(f"\n{'=' * 60}", "white"))
-        print(colorText("📊 Local Approval Summary", "cyan"))
+        print(colorText("ðŸ“Š Local Approval Summary", "cyan"))
        print(colorText("=" * 60, "white"))

-        print(colorText(f"✓ Successfully processed: {success_count}", "green"))
+        print(colorText(f"âœ“ Successfully processed: {success_count}", "green"))

        if failure_count > 0:
-            print(colorText(f"✗ Failed: {failure_count}", "red"))
+            print(colorText(f"âœ— Failed: {failure_count}", "red"))

-        print(colorText(f"\n📦 Batch ID: {batch_id}", "cyan"))
-        print(colorText(f"⏱️  Duration: {duration_label}", "cyan"))
+        print(colorText(f"\nðŸ“¦ Batch ID: {batch_id}", "cyan"))
+        print(colorText(f"â±ï¸  Duration: {duration_label}", "cyan"))

        print(colorText("=" * 60, "white"))
-        print(colorText("\n💡 Next Steps:", "yellow"))
-        print(colorText("  • Agents have been moved to audit policies", "white"))
-        print(colorText("  • Local approvals are active", "white"))
+        print(colorText("\nðŸ’¡ Next Steps:", "yellow"))
+        print(colorText("  â€¢ Agents have been moved to audit policies", "white"))
+        print(colorText("  â€¢ Local approvals are active", "white"))
        print(
            colorText(
-                f"  • Agents will return to enforcement after {duration_label}", "white"
+                f"  â€¢ Agents will return to enforcement after {duration_label}",
+                "white",
            )
        )
        print(colorText("=" * 60 + "\n", "white"))
@@ -25,7 +25,7 @@ import pandas as pd
 from models.execution import ExecutionHistoryRecord
 from models.policy import Allowlist, Policy
 from services.API import AirlockAPIWrapper
-from utils.configmanager import get_protected_value, load_env, load_env_json
+from utils.configmanager import get_system_list, get_system_value, load_env
 from utils.selector import Selector
 from utils.utils import (
    areYouSure,
@@ -88,7 +88,7 @@ def sortHashes(
 ):
    working_dir = load_env("WORKING_DIR")
    history_days = Selector.select_value(
-        prompt="Enter how many days of history to pull (1–150): ",
+        prompt="Enter how many days of history to pull (1â€“150): ",
        value_type=int,
        valid_range=(1, 150),
    )
@@ -157,7 +157,7 @@ def buildPathsandPublishers(selected_policies: List[Policy], split):
        f"{working_dir}\\Approved\\{selected_policies[0].name}_approved_executions.csv"
    )
    path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_needs_review_executions.csv"
-    path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type=int)
+    path_exclusion_constant = get_system_value("PATH_EXCLUSION_CONST", cast_type=int)

    if os.path.exists(path1):
        df1 = pd.read_csv(path1)
@@ -227,7 +227,7 @@ def buildPathsandPublishers(selected_policies: List[Policy], split):
                all_approved_hashes["publisher"] != "Not Signed"
            ].drop_duplicates(subset=["publisher"])
            # Remove Bad publisher if somehow they made it this far
-            pattern = regulator(load_env_json("BAD_PUBLISHERS", "[]"))
+            pattern = regulator(get_system_list("BAD_PUBLISHERS"))
            publist = publist[~publist["publisher"].str.contains(pattern, na=False)]
            publist = publist[["publisher"]]
            publist.sort_values(by="publisher", inplace=True)
@@ -313,7 +313,7 @@ def buildPreflights(selected_policies: List[Policy]):


 def splitFilepathsGrouped(df, path_exclusion_constant, col="filename"):
-    min_files_for_path = get_protected_value("MIN_FILES_FOR_PATH", cast_type=int)
+    min_files_for_path = get_system_value("MIN_FILES_FOR_PATH", cast_type=int)

    def clean_split(path):
        if not isinstance(path, (str, bytes, os.PathLike)):
@@ -385,8 +385,8 @@ def calculatePath(approved_hashes, path_exclusion_constant, split):
    else:
        dfs_by_policy = [approved_hashes]

-    badpathparts = load_env_json("BAD_PATH_PARTS", "[]")
-    min_files_for_path = get_protected_value("MIN_FILES_FOR_PATH", cast_type=int)
+    badpathparts = get_system_list("BAD_PATH_PARTS")
+    min_files_for_path = get_system_value("MIN_FILES_FOR_PATH", cast_type=int)

    processed_dfs = []

@@ -655,7 +655,7 @@ def section_header(title):

 def printEnforceChecklist(selected_policies, destination_policy, destination_allowlist):
    working_dir = load_env("WORKING_DIR")
-    section_header("🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒")
+    section_header("ðŸ› ï¸ ðŸ”’ Prepare to Enforce Policy ðŸ› ï¸ ðŸ”’")
    print(
        colorText(
            "\nSequentially follow these steps to prepare a policy for enforcement:",
@@ -670,11 +670,11 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
        )
    )
    if not selected_policies:
-        print(colorText("  [✗]  No policies have been chosen", "red"))
+        print(colorText("  [âœ—]  No policies have been chosen", "red"))
    else:
        print(colorText("The following policies have been chosen:", "green"))
        for policy in selected_policies:
-            print(colorText(f"  [✓] {policy.name}", "green"))
+            print(colorText(f"  [âœ“] {policy.name}", "green"))

    # Step 2: Destination Policy and Allowlist
    print(
@@ -683,22 +683,22 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
    if destination_policy:
        print(
            colorText(
-                f"  [✓] {destination_policy[0].name} has been selected as the destination policy",
+                f"  [âœ“] {destination_policy[0].name} has been selected as the destination policy",
                "green",
            )
        )
    else:
-        print(colorText("  [✗]  No destination policy has been chosen", "red"))
+        print(colorText("  [âœ—]  No destination policy has been chosen", "red"))

    if destination_allowlist:
        print(
            colorText(
-                f"  [✓] {destination_allowlist[0].name} has been selected as allowlist",
+                f"  [âœ“] {destination_allowlist[0].name} has been selected as allowlist",
                "green",
            )
        )
    else:
-        print(colorText("  [✗]  No allowlist has been chosen", "red"))
+        print(colorText("  [âœ—]  No allowlist has been chosen", "red"))

    # Step 3: Data Preparation
    print(
@@ -713,9 +713,9 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
        print(
            colorText(
                (
-                    "  [✓] Data has been fetched"
+                    "  [âœ“] Data has been fetched"
                    if os.path.exists(review_path)
-                    else "  [✗] Data has not been fetched"
+                    else "  [âœ—] Data has not been fetched"
                ),
                "green" if os.path.exists(review_path) else "red",
            )
@@ -723,7 +723,7 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
    else:
        print(
            colorText(
-                "  [✗]  No policies selected, cannot check data fetch status", "red"
+                "  [âœ—]  No policies selected, cannot check data fetch status", "red"
            )
        )

@@ -756,9 +756,9 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
        print(
            colorText(
                (
-                    "  [✓]  Reviewed hashes have been loaded"
+                    "  [âœ“]  Reviewed hashes have been loaded"
                    if os.path.exists(approved_path)
-                    else "  [✗]  Reviewed hashes have not been loaded"
+                    else "  [âœ—]  Reviewed hashes have not been loaded"
                ),
                "green" if os.path.exists(approved_path) else "red",
            )
@@ -766,9 +766,9 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
        print(
            colorText(
                (
-                    "  [✓]  Path review list created"
+                    "  [âœ“]  Path review list created"
                    if os.path.exists(second_review_path)
-                    else "  [✗]  Path review list has not been created"
+                    else "  [âœ—]  Path review list has not been created"
                ),
                "green" if os.path.exists(second_review_path) else "red",
            )
@@ -776,7 +776,7 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
    else:
        print(
            colorText(
-                "  [✗]  No policies selected, cannot check reviewed hashes or path list",
+                "  [âœ—]  No policies selected, cannot check reviewed hashes or path list",
                "red",
            )
        )
@@ -812,9 +812,9 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
        print(
            colorText(
                (
-                    "  [✓]  Reviewed path list detected"
+                    "  [âœ“]  Reviewed path list detected"
                    if os.path.exists(reviewed_path)
-                    else "  [✗]  Path review list has not been detected"
+                    else "  [âœ—]  Path review list has not been detected"
                ),
                "green" if os.path.exists(reviewed_path) else "red",
            )
@@ -825,9 +825,9 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
        print(
            colorText(
                (
-                    "  [✓]  Preflight Path Exclusion List has been generated"
+                    "  [âœ“]  Preflight Path Exclusion List has been generated"
                    if preflight_ready
-                    else "  [✗]  Preflight Path Exclusion List has not been generated"
+                    else "  [âœ—]  Preflight Path Exclusion List has not been generated"
                ),
                "green" if preflight_ready else "red",
            )
@@ -835,7 +835,7 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
    else:
        print(
            colorText(
-                "  [✗]  No policies selected, cannot check preflight status", "red"
+                "  [âœ—]  No policies selected, cannot check preflight status", "red"
            )
        )

@@ -866,5 +866,5 @@ def printEnforceChecklist(selected_policies, destination_policy, destination_all
    print(colorText("     Apply approved hashes to allowlist", "cyan"))

    # Utility Options
-    print(colorText("F. 📂  - Open Working Directory", "cyan"))
-    print(colorText("B. 🔚  - Back", "cyan"))
+    print(colorText("F. ðŸ“‚  - Open Working Directory", "cyan"))
+    print(colorText("B. ðŸ”š  - Back", "cyan"))
@@ -1,134 +0,0 @@
-#    Copyright (C) 2025 James Brotosky, Brandon Wickline
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import datetime
-import logging
-
-import dotenv
-import pandas as pd
-
-from flows.prepPolicy import selectPolicies
-from services.API import AirlockAPIWrapper
-from services.policyhandler import getPolicyInfo
-from utils.configmanager import load_env
-from utils.selector import Selector
-from utils.utils import colorText, get_sanitized_input
-
-logger = logging.getLogger(__name__)
-
-
-dotenv.load_dotenv()
-
-
-def findQuietAgents(api: AirlockAPIWrapper):
-    working_dir = load_env("WORKING_DIR")
-    # Get policy selection and agent list
-    selected_policy = selectPolicies(api, False)
-    if selected_policy:
-        agents = api.agents_find_by_group(selected_policy[0].groupid)
-
-        # Prompt user for history range
-        history_days = Selector.select_value(
-            prompt="Enter how many days of history to pull (1–150): ",
-            value_type=int,
-            valid_range=(1, 150),
-        )
-        required_quiet = Selector.select_value(
-            prompt="Enter how many days without an untrusted execution before these are considered ready for enforcement? (1–365): ",
-            value_type=int,
-            valid_range=(1, 150),
-        )
-
-        confirm = Selector.confirm(
-            f"Do you wish to proceed to pull history for {selected_policy[0].name}? Y/N :  "
-        )
-        # Get execution history as a DataFrame
-        if confirm:
-            policy_exec_history = getPolicyInfo(
-                api, selected_policy[0], [1, 2, 6, 7], history_days
-            )
-
-            if policy_exec_history.empty:
-                logging.info(
-                    "No execution history found for the selected policy and time range."
-                )
-                get_sanitized_input("Press enter to continue")
-                return
-
-            # Convert 'datetime' column to timezone-aware datetime objects
-            policy_exec_history["datetime"] = pd.to_datetime(
-                policy_exec_history["datetime"], format="%Y-%m-%dT%H:%M:%SZ", utc=True
-            )
-
-            # Get current UTC time
-            now = datetime.datetime.now(datetime.timezone.utc)
-
-            # Calculate days ago
-            policy_exec_history["days_ago"] = policy_exec_history["datetime"].apply(
-                lambda dt: (now - dt).days
-            )
-
-            # Count total executions per hostname
-            hostname_counts = policy_exec_history["hostname"].value_counts()
-
-            # Map execution counts to agents
-            agents["execution_count"] = (
-                agents["hostname"].map(hostname_counts).fillna(0).astype(int)
-            )
-
-            # Find most recent execution per hostname
-            most_recent_exec = policy_exec_history.sort_values(
-                by="days_ago"
-            ).drop_duplicates(subset="hostname", keep="first")
-
-            # Map most recent execution age to agents
-            agents["days_since"] = agents["hostname"].map(
-                most_recent_exec.set_index("hostname")["days_ago"]
-            )
-
-            # Check for enforcement readiness
-            agents["required_quiet"] = required_quiet
-            agents["enforce_ready"] = agents["days_since"].apply(
-                lambda x: True if pd.isna(x) or x > required_quiet else False
-            )
-
-            # Sort agents by execution count and hostname
-            agents = agents.sort_values(
-                by=["execution_count", "hostname"], ascending=[True, True]
-            )
-
-            # Save to CSV
-            filename = f"{working_dir}\\{selected_policy[0].name}_agents_last_{history_days}_days.csv"
-            logging.debug(f"Saving CSV to {filename}")
-            print(colorText(f"Saving CSV to {filename}", "green"))
-            agents.to_csv(filename, index=False)
-
-            # Summary statistics
-            total_agents = len(agents)
-            ready_agents = agents["enforce_ready"].sum()
-            not_ready_agents = total_agents - ready_agents
-            ready_percentage = (ready_agents / total_agents) * 100
-
-            # Print results
-
-            message = (
-                f"Total agents: {total_agents}\n"
-                f"Agents marked as 'enforce_ready': {ready_agents}\n"
-                f"Agents not ready: {not_ready_agents}\n"
-                f"Percentage ready for enforcement: {ready_percentage:.2f}%"
-            )
-            logger.debug(message)
-            colorText(message, "green")
-    get_sanitized_input("Press enter to continue")