From 908316dc34bfe614256e62030bd0c7f9873e417f Mon Sep 17 00:00:00 2001 From: Zarithas Date: Mon, 20 Oct 2025 16:42:37 -0400 Subject: [PATCH] Bugfixes + QOL --- flows/prepPolicy.py | 56 ++++++------ models/execution.py | 6 +- utils/menus.py | 30 +++---- utils/utils.py | 202 +++++++++++++------------------------------- 4 files changed, 107 insertions(+), 187 deletions(-) diff --git a/flows/prepPolicy.py b/flows/prepPolicy.py index 6caedf8..d830931 100644 --- a/flows/prepPolicy.py +++ b/flows/prepPolicy.py @@ -100,13 +100,13 @@ def sortHashes( "needs_review": needs_review, "approved": approved, "unapproved": unapproved, - "unknown" : unknown + "leftover" : unknown } for label, records in categories.items(): - csv_path = f"{working_dir}\\Needs_Review\\Review_First\\{label}_executions.csv" - html_path = f"{working_dir}\\Needs_Review\\HTML\\{label}.html" + csv_path = f"{working_dir}\\Needs_Review\\Review_First\\{selected_policies[0].name}_{label}_executions.csv" + html_path = f"{working_dir}\\Needs_Review\\HTML\\{selected_policies[0].name}_{label}.html" # Convert ExecutionHistoryRecord objects to dictionaries df = pd.DataFrame([r.__dict__ for r in records]) @@ -125,13 +125,13 @@ def sortHashes( logger.info(f"Generated HTML report at {html_path}") -def buildPathsandPublishers(split): +def buildPathsandPublishers(selected_policies: List[Policy], split): working_dir = load_env("WORKING_DIR") df1 = pd.DataFrame() df2 = pd.DataFrame() all_approved_hashes = pd.DataFrame() - path1 = f"{working_dir}\\Approved\\approved_executions.csv" - path2 = f"{working_dir}\\Approved\\needs_review_executions.csv" + path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_approved_executions.csv" + path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_needs_review_executions.csv" if os.path.exists(path1): df1 = pd.read_csv(path1) @@ -149,10 +149,10 @@ def buildPathsandPublishers(split): logger.debug(all_approved_hashes.head) else: all_approved_hashes = pd.concat([df1, df2], ignore_index=True) - if "filename_exec" in all_approved_hashes.columns: - all_approved_hashes = all_approved_hashes.sort_values(by="filename_exec") + if "filename" in all_approved_hashes.columns: + all_approved_hashes = all_approved_hashes.sort_values(by="filename") else: - logger.warning("Warning: 'filename_exec' column not found in concatenated DataFrame.") + logger.warning("Warning: 'filename' column not found in concatenated DataFrame.") if not all_approved_hashes.empty: primary_path_exclusions = calculatePath( @@ -176,25 +176,25 @@ def buildPathsandPublishers(split): logger.debug("Preparing to sort dataframes") for name, df in dataframes.items(): logger.debug(f" DataFrame headers: {list(df.columns)}") - if name == "hashes_to_add": df.sort_values(by="filename_exec", inplace=True) + if name == "hashes_to_add": df.sort_values(by="filename", inplace=True) else: df.sort_values(by="longestcfp", inplace=True) - df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{name}.csv", index=False) - formatHTML(df, f"{working_dir}\\Needs_Review\\HTML\\{name}.html") + df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_{name}.csv", index=False) + formatHTML(df, f"{working_dir}\\Needs_Review\\HTML\\{selected_policies[0].name}_{name}.html") if not all_approved_hashes.empty: # Drop all not signed, only keep unique values publist = all_approved_hashes[ - all_approved_hashes["publisher_hash"] != "Not Signed" - ].drop_duplicates(subset=["publisher_hash"]) + all_approved_hashes["publisher"] != "Not Signed" + ].drop_duplicates(subset=["publisher"]) # Remove Bad publisher if somehow they made it this far pattern = regulator(load_env_json("BAD_PUBLISHERS","[]")) - publist = publist[~publist["publisher_hash"].str.contains(pattern, na=False)] - publist = publist[["publisher_hash"]] - publist.sort_values(by="publisher_hash", inplace=True) - publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\publishers.csv", index=False) + publist = publist[~publist["publisher"].str.contains(pattern, na=False)] + publist = publist[["publisher"]] + publist.sort_values(by="publisher", inplace=True) + publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_publishers.csv", index=False) -def buildPreflights(): +def buildPreflights(selected_policies: List[Policy]): working_dir = load_env("WORKING_DIR") df1 = pd.DataFrame() @@ -202,10 +202,10 @@ def buildPreflights(): approved_hashes = pd.DataFrame() approved_publishers = pd.DataFrame() - hash = f"{working_dir}\\Approved\\hashes_to_add.csv" - path1 = f"{working_dir}\\Approved\\primary_Paths.csv" - path2 = f"{working_dir}\\Approved\\secondary_Paths.csv" - publishers = f"{working_dir}\\Approved\\publishers.csv" + hash = f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv" + path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv" + path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_secondary_Paths.csv" + publishers = f"{working_dir}\\Approved\\{selected_policies[0].name}_publishers.csv" if os.path.exists(hash): approved_hashes = pd.read_csv(hash) @@ -240,11 +240,11 @@ def buildPreflights(): for name, df in dataframes.items(): logger.debug(f" DataFrame headers: {list(df.columns)}") if name == "approved_paths":df.sort_values(by="longestcfp", inplace=True) - elif name == "approved_hashes":df.sort_values(by="filename_exec", inplace=True) - elif name == "approved_publishers" : df.sort_values(by="publisher_hash", inplace=True) + elif name == "approved_hashes":df.sort_values(by="filename", inplace=True) + elif name == "approved_publishers" : df.sort_values(by="publisher", inplace=True) - df.to_csv(f"{working_dir}\\Preflight\\{name}.csv", index=False) - formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{name}.html") + df.to_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_{name}.csv", index=False) + formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{selected_policies[0].name}_{name}.html") def splitFilepathsGrouped(df, col="filename"): path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type= int) @@ -319,7 +319,7 @@ def calculatePath(approved_hashes, split): processed_dfs = [] for df in dfs_by_policy: - haslcp = splitFilepathsGrouped(df, "filename_exec") + haslcp = splitFilepathsGrouped(df, "filename") haslcp = haslcp.drop_duplicates() forbidden = regulator(badpathparts, True) diff --git a/models/execution.py b/models/execution.py index 1ef71d1..a4231fc 100644 --- a/models/execution.py +++ b/models/execution.py @@ -442,7 +442,9 @@ class ExecutionHistoryRecord: needs_review = [] unknown = [] - for record in executions: + sorted_executions = sorted(executions, key=lambda x: x.filename) + + for record in sorted_executions: decision = getattr(record.hash_obj, "at_decision", None) if decision == "approved": approved.append(record) @@ -453,7 +455,7 @@ class ExecutionHistoryRecord: else: unknown.append(record) - logger.info(f"[ExecutionHistoryRecord] Sorted {len(executions)} records by hash_obj.at_decision:") + logger.info(f"[ExecutionHistoryRecord] Sorted {len(sorted_executions)} records by hash_obj.at_decision:") logger.info(f" Approved: {len(approved)}") logger.info(f" Unapproved: {len(unapproved)}") logger.info(f" Needs Review: {len(needs_review)}") diff --git a/utils/menus.py b/utils/menus.py index e10f8c3..404dfbb 100644 --- a/utils/menus.py +++ b/utils/menus.py @@ -133,31 +133,31 @@ def menu_policy_enforce(api: AirlockAPIWrapper): ) elif choice == "4": - if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\approved_executions.csv"): - buildPathsandPublishers(False) + if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\{selected_policies[0].name}_approved_executions.csv"): + buildPathsandPublishers(selected_policies, False) else: print("File not found. Please make sure it's saved correctly and try again.") elif choice == "5": - if os.path.exists(f"{working_dir}\\Approved\\hashes_to_add.csv") and os.path.exists( - f"{working_dir}\\Approved\\primary_Paths.csv" + if os.path.exists(f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv") and os.path.exists( + f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv" ): - buildPreflights() + buildPreflights(selected_policies) else: print("File not found. Please make sure it's saved correctly and try again.") elif choice == "6": if ( - os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv") - and os.path.exists(f"{working_dir}\\Preflight\\approved_hashes.csv") + os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv") + and os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv") and destination_policy and destination_allowlist ): print(colorText("These path exclusions would be added to:", "yellow")) print(destination_policy) - pathexclusions = pd.read_csv(f"{working_dir}\\Preflight\\approved_paths.csv") - hashes = pd.read_csv(f"{working_dir}\\Preflight\\approved_hashes.csv") + pathexclusions = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv") + hashes = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv") unique_combinations = pathexclusions[["longestcfp", "file_extension"]].drop_duplicates() @@ -170,14 +170,14 @@ def menu_policy_enforce(api: AirlockAPIWrapper): print(processed_paths) print(colorText("These publishers would added", "yellow")) - if os.path.exists(f"{working_dir}\\Preflight\\approved_publishers.csv"): - publishers = pd.read_csv(f"{working_dir}\\Preflight\\approved_hashes.csv") + if os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_publishers.csv"): + publishers = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv") if publishers.empty: print(colorText("The publishers list is empty.", "red")) else: processed_publishers = ( - publishers[publishers["publisher_hash"] != "Not Signed"] - ["publisher_hash"] + publishers[publishers["publisher"] != "Not Signed"] + ["publisher"] .drop_duplicates() .tolist() ) @@ -194,9 +194,9 @@ def menu_policy_enforce(api: AirlockAPIWrapper): else: # Log which condition(s) failed missing_items = [] - if not os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv"): + if not os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv"): missing_items.append("approved_paths.csv not found") - if not os.path.exists(f"{working_dir}\\Preflight\\approved_hashes.csv"): + if not os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv"): missing_items.append("approved_hashes.csv not found") if not destination_policy: missing_items.append("destination_policy is empty or None") diff --git a/utils/utils.py b/utils/utils.py index 6803411..fdb1d66 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -187,179 +187,97 @@ def displayIntro(): def printEnforceChecklist(selected_policies, destination_policy, destination_allowlist): working_dir = load_env("WORKING_DIR") - print( - colorText( - "\n --------------------------------------------------------------------", - "cyan", - ) - ) - print( - colorText( - " ------------- 🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒 ------------------", - "cyan", - ) - ) - print( - colorText( - " --------------------------------------------------------------------", - "cyan", - ) - ) - print( - colorText( - "\nSequentually follow these steps to prepare a policy for enforcement:", - "white", - ) - ) - print( - colorText( - "\n1. Choose which originating policy or policies to move to enforcement", - "cyan", - ) - ) + def section_header(title): + print(colorText("\n --------------------------------------------------------------------", "cyan")) + print(colorText(f" ------------- {title} -------------", "cyan")) + print(colorText(" --------------------------------------------------------------------", "cyan")) + + section_header("🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒") + print(colorText("\nSequentially follow these steps to prepare a policy for enforcement:", "white")) + + # Step 1: Originating Policies + print(colorText("\n1. Choose which originating policy or policies to move to enforcement", "cyan")) if not selected_policies: print(colorText(" [✗] No policies have been chosen", "red")) else: - print(colorText("The following policies have been choosen:", "green")) + print(colorText("The following policies have been chosen:", "green")) for policy in selected_policies: print(colorText(f" [✓] {policy.name}", "green")) - print(colorText("2. Choose the destination policy and allowlist", "cyan")) - - if not destination_policy: - print(colorText(" [✗] No destination policy has been chosen", "red")) - elif destination_policy: + # Step 2: Destination Policy and Allowlist + print(colorText("2. Choose the destination policy and allowlist", "cyan")) + if destination_policy: print(colorText(f" [✓] {destination_policy[0].name} has been selected as the destination policy", "green")) + else: + print(colorText(" [✗] No destination policy has been chosen", "red")) - - - if not destination_allowlist: + if destination_allowlist: + print(colorText(f" [✓] {destination_allowlist[0].name} has been selected as allowlist", "green")) + else: print(colorText(" [✗] No allowlist has been chosen", "red")) - elif destination_allowlist: - print( - colorText( - f" [✓] {destination_allowlist[0].name} has been selected as allowlist", - "green", - ) - ) - - - print( - colorText( - "3. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", - "cyan", - ) - ) - if not selected_policies: - print(colorText(" [✗] No policies have been chosen", "red")) + # Step 3: Data Preparation + print(colorText("3. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", "cyan")) + if selected_policies: + policy_id = selected_policies[0].name + review_path = f"{working_dir}\\Needs_Review\\Review_First\\{policy_id}_approved_executions.csv" + print(colorText(" [✓] Data has been fetched" if os.path.exists(review_path) else " [✗] Data has not been fetched", "green" if os.path.exists(review_path) else "red")) else: - if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\approved_executions.csv"): - print(colorText(" [✓] Data has been fetched", "green")) - else: - print(colorText(" [✗] Data has not been fetched", "red")) + print(colorText(" [✗] No policies selected, cannot check data fetch status", "red")) + # Step 4: Manual Review print(colorText("4. Manually review the files:", "cyan")) - print( - colorText( - " 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\unknown_hashes.csv'\n", - "cyan", - ) - ) - print( - colorText( - " Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.", - "cyan", - ) - ) - print( - colorText( - " If metarules need to be created, please make note of them, and remove the row from the csv.", - "cyan", - ) - ) - print( - colorText( - " When complete, save both csv files to the directory 'approved' and choose this option.", - "cyan", - ) - ) - print( - colorText( - " This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", - "cyan", - ) - ) - - if os.path.exists(f"{working_dir}\\Approved\\approved_executions.csv"): - print(colorText(" [✓] Reviewed hashes have been loaded", "green")) + if selected_policies: + policy_id = selected_policies[0].name + print(colorText(f" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\{policy_id}_unknown_hashes.csv'\n", "cyan")) else: - print(colorText(" [✗] Reviewed hashes have not been loaded", "red")) + print(colorText(" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\_unknown_hashes.csv'\n", "cyan")) - if os.path.exists( - f"{working_dir}\\Needs_Review\\Review_Second\\primary_Paths.csv", - ): - print(colorText(" [✓] Path review list created", "green")) + print(colorText(" Remove the rows containing hashes you do not approve of...", "cyan")) + print(colorText(" When complete, save both csv files to the directory 'approved' and choose this option.", "cyan")) + print(colorText(" This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan")) + + if selected_policies: + policy_id = selected_policies[0].name + approved_path = f"{working_dir}\\Approved\\{policy_id}_approved_executions.csv" + second_review_path = f"{working_dir}\\Needs_Review\\Review_Second\\{policy_id}_primary_Paths.csv" + print(colorText(" [✓] Reviewed hashes have been loaded" if os.path.exists(approved_path) else " [✗] Reviewed hashes have not been loaded", "green" if os.path.exists(approved_path) else "red")) + print(colorText(" [✓] Path review list created" if os.path.exists(second_review_path) else " [✗] Path review list has not been created", "green" if os.path.exists(second_review_path) else "red")) else: - print(colorText(" [✗] Path review list has not been created", "red")) + print(colorText(" [✗] No policies selected, cannot check reviewed hashes or path list", "red")) - print( - colorText( - "5. Manually review the files \n 'prepare_policy\\needs_approved\\primary_Paths.csv'\n 'prepare_policy\\needs_approved\\secondary_Paths.csv'", - "cyan", - ) - ) - print( - colorText( - " Remove the rows containing path exclusions you do not approve of. The secondary list can be not added at all if nothing is useful", - "cyan", - ) - ) + # Step 5: Path Review + print(colorText("5. Manually review the files \n 'prepare_policy\\needs_approved\\primary_Paths.csv'\n 'prepare_policy\\needs_approved\\secondary_Paths.csv'", "cyan")) + print(colorText(" Remove the rows containing path exclusions you do not approve of...", "cyan")) print(colorText(" When complete, save the csv file to the directory 'approved'", "cyan")) - print( - colorText( - " Do the same process with the list of publishers forthe same directories", - "cyan", - ) - ) + print(colorText(" Do the same process with the list of publishers for the same directories", "cyan")) print(colorText(" Preflight Lists will be generated", "cyan")) - if os.path.exists( - f"{working_dir}\\Approved\\primary_Paths.csv", - ): - print(colorText(" [✓] Reviewed path list detected", "green")) + if selected_policies: + policy_id = selected_policies[0].name + reviewed_path = f"{working_dir}\\Approved\\{policy_id}_primary_Paths.csv" + preflight_paths = f"{working_dir}\\Preflight\\{policy_id}_approved_paths.csv" + preflight_hashes = f"{working_dir}\\Preflight\\{policy_id}_approved_hashes.csv" + print(colorText(" [✓] Reviewed path list detected" if os.path.exists(reviewed_path) else " [✗] Path review list has not been detected", "green" if os.path.exists(reviewed_path) else "red")) + preflight_ready = os.path.exists(preflight_paths) and os.path.exists(preflight_hashes) + print(colorText(" [✓] Preflight Path Exclusion List has been generated" if preflight_ready else " [✗] Preflight Path Exclusion List has not been generated", "green" if preflight_ready else "red")) else: - print(colorText(" [✗] Path review list has not been detected", "red")) - - if os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv") and os.path.exists( - f"{working_dir}\\Preflight\\approved_hashes.csv" - ): - print(colorText(" [✓] Preflight Path Exclusion List has been generated", "green")) - else: - print(colorText(" [✗] Preflight Path Exclusion List has not been generated", "red")) + print(colorText(" [✗] No policies selected, cannot check preflight status", "red")) + # Final Steps print(colorText("6. Test ------------------------------------------------------", "cyan")) print(colorText(" Print rather than apply selected data.", "cyan")) print(colorText("7. Liftoff ------------------------------------------------------", "cyan")) - print( - colorText( - " Apply path exclusions according to allowed and approved paths", - "cyan", - ) - ) + print(colorText(" Apply path exclusions according to allowed and approved paths", "cyan")) print(colorText(" Apply signed or attested hashes to Parent Allow List", "cyan")) print(colorText(" Apply approved, but unsigned hashes to the Child Allow List", "cyan")) - print( - colorText( - "R. Remove/Reset Generated data - will prompt to allow keeping execution history", - "cyan", - ) - ) + # Utility Options + print(colorText("R. Remove/Reset Generated data - will prompt to allow keeping execution history", "cyan")) print(colorText("F. 📂 - Open Working Directory", "cyan")) - print(colorText("Q. 🔚 - Quit", "cyan")) + print(colorText("B. 🔚 - Back", "cyan")) def areYouSure():