From d767ba962136bd755279d04ab1fd7d8d4bb88912 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 25 Aug 2025 14:43:37 -0400 Subject: [PATCH] Everything Ready for the API Calls to approve in airlock --- AirlockTools.py | 94 +++++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/AirlockTools.py b/AirlockTools.py index 7c1060a..20cfaf2 100644 --- a/AirlockTools.py +++ b/AirlockTools.py @@ -19,10 +19,10 @@ import utils.getdeviceevents import utils.allowlist import utils.hashfunctions import utils.pathfunctions -import utils.allowfunctions import utils.colortext as ct import urllib3 import pandas as pd +import ast urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) dotenv.load_dotenv() @@ -117,6 +117,7 @@ def menu_prepare_to_enforce(): first_policy = " " second_policy = " " + #If the directorys where we're going to store our output dont exist, make them. if not os.path.exists("dataframe_html"): os.makedirs("dataframe_html") @@ -164,38 +165,32 @@ def menu_prepare_to_enforce(): else: print(ct.colorText(" [✗] This step has not been completed","red")) - print(ct.colorText("5. Determine if path exclusions are possible", "cyan")) - - if os.path.exists(f"dataframe_csv\\df_path_eligible_{first_policy}_{second_policy}.csv") == True: - print(ct.colorText(" [✓] This step has been completed","green")) - else: - print(ct.colorText(" [✗] This step has not been completed", "red")) - - print(ct.colorText("6. Categorize your hashes ", "cyan")) + print(ct.colorText("5. Categorize your hashes ", "cyan")) if os.path.isfile(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv"): print(ct.colorText(" [✓] This step has been completed","green")) else: print(ct.colorText(" [✗] This step has not been completed","red")) - print(ct.colorText(f"7. Manually review the files \\dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv and dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", "cyan")) + print(ct.colorText(f"6. Manually review the files \\dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv and dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", "cyan")) print(ct.colorText(" Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.", "cyan")) print(ct.colorText(" If metarules need to be created, please make note of them, and remove the row from the csv.", "cyan")) - print(ct.colorText(" When complete, move both csv files to the directory 'manuallyapproved' and choose this option to combine these approved hashes with the automatically approved hashes", "cyan")) + print(ct.colorText(" When complete, save both csv files to the directory 'manuallyapproved' and choose this option to combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan")) - if os.path.isfile(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv"): + if os.path.isfile(f"dataframe_csv\\df_paths_needing_review_{first_policy}_{second_policy}.csv"): print(ct.colorText(" [✓] This step has been completed","green")) else: print(ct.colorText(" [✗] This step has not been completed","red")) - """ - print(ct.colorText("7. Compare potential path exclusions with allowed hashes", "cyan")) - - if os.path.exists(f"dataframe_csv\\df_allowed_paths_{first_policy}_{second_policy}.csv") == True: + print(ct.colorText(f"7. Manually review the file df_paths_needing_review_{first_policy}_{second_policy}.csv", "cyan")) + print(ct.colorText(" Remove the rows containing path exclusions you do not approve of" , "cyan")) + print(ct.colorText(" When complete, save the csv file to the directory 'manuallyapproved' and choose this option to generate the proposed list of changes", "cyan")) + + if os.path.isfile(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv"): print(ct.colorText(" [✓] This step has been completed","green")) else: print(ct.colorText(" [✗] This step has not been completed","red")) - """ + print(ct.colorText("Q. Quit", "cyan")) @@ -235,11 +230,14 @@ def menu_prepare_to_enforce(): if second_policy is first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv"): df1 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{first_policy}.csv") df_aggregated_combo = df1 + df_aggregated_combo.to_html(f"dataframe_html\\df_aggregated_combo_{first_policy}_{second_policy}.html", index=False) df_aggregated_combo.to_csv(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv", index=False) + elif os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv"): df1 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{first_policy}.csv") df2 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{second_policy}.csv") + df_aggregated_combo = pd.concat([df1 , df2], ignore_index=True) df_aggregated_combo.to_html(f"dataframe_html\\df_aggregated_combo_{first_policy}_{second_policy}.html", index=False) df_aggregated_combo.to_csv(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv", index=False) @@ -249,43 +247,63 @@ def menu_prepare_to_enforce(): elif choice == "4": if os.path.exists(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv"): df_augmented = utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv")) + df_augmented.to_html(f"dataframe_html\\df_augmented_combo_{first_policy}_{second_policy}.html", index=False) df_augmented.to_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv", index=False) + print(ct.colorText(f"Hash reputation info added to dataframe","green")) else: print(ct.colorText(f"Please combine your data with step 3 prior to attempting this step","red")) elif choice == "5": - if os.path.exists(f"dataframe_html\\df_augmented_combo_{first_policy}_{second_policy}.html"): - path_eligible, path_ineligible = utils.pathfunctions.filepathInitialGroup(pd.read_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv")) - path_eligible.to_html(f"dataframe_html\\df_path_eligible_{first_policy}_{second_policy}.html", index=False) - path_eligible.to_csv(f"dataframe_csv\\df_path_eligible_{first_policy}_{second_policy}.csv", index=False) - path_ineligible.to_html(f"dataframe_html\\df_path_ineligible_{first_policy}_{second_policy}.html", index=False) - path_ineligible.to_csv(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv", index=False) - print(ct.colorText(f"Eligible paths determined","green")) - else: - print(ct.colorText(f"Please Augment your data with hash threat info using step 4 prior to attempting this step","red")) - - elif choice == "6": if os.path.exists(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv"): categorized = utils.hashfunctions.categorizeHashes(pd.read_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv"), threat_tolerance_constant, badpublisherlist) + categorized[0].to_html(f"dataframe_html\\df_hashes_needing_approval_{first_policy}_{second_policy}.html", index=False) categorized[0].to_csv(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False) + categorized[1].to_html(f"dataframe_html\\df_automatically_approved_hashes_{first_policy}_{second_policy}.html", index=False) categorized[1].to_csv(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False) + categorized[2].to_html(f"dataframe_html\\df_unapproved_hashes__{first_policy}_{second_policy}.html", index=False) categorized[2].to_csv(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv", index=False) + print(ct.colorText(f"Hashes have been categorized","green")) else: print(ct.colorText(f"Please Augment your data with hash threat info using step 4 prior to attempting this step","red")) - elif choice == "7": - if os.path.isfile(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv"): + elif choice == "6": + if os.path.exists(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.exists(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv"): df1 = tryToReadCSV(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") df2 = tryToReadCSV(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv") + df_all_approved_hashes = pd.concat([df1 , df2], ignore_index=True) - df_aggregated_combo.to_html(f"dataframe_html\\df_all_approved_hashes_{first_policy}_{second_policy}.html", index=False) - df_aggregated_combo.to_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv", index=False) + df_all_approved_hashes.to_html(f"dataframe_html\\df_all_approved_hashes_{first_policy}_{second_policy}.html", index=False) + df_all_approved_hashes.to_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv", index=False) + + + df_paths_needing_review, df_path_ineligible = utils.pathfunctions.filepathInitialGroup(pd.read_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv")) + + df_paths_needing_review.to_html(f"dataframe_html\\df_paths_needing_review_{first_policy}_{second_policy}.html", index=False) + df_paths_needing_review.to_csv(f"dataframe_csv\\df_paths_needing_review_{first_policy}_{second_policy}.csv", index=False) + + df_path_ineligible.to_html(f"dataframe_html\\df_path_ineligible_{first_policy}_{second_policy}.html", index=False) + df_path_ineligible.to_csv(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv", index=False) + + print(ct.colorText(f"Eligible paths determined","green")) + else: + print(ct.colorText(f"Please manually approve hashes prior to this step","red")) + + + elif choice == "7": + if os.path.exists(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv"): + df1 = tryToReadCSV(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv") + df2 = tryToReadCSV(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv") + print(df1['grouped_directory']) + print(df2['sha256']) + + + elif choice == "Q": @@ -293,17 +311,7 @@ def menu_prepare_to_enforce(): else: print(ct.colorText("Invalid choice. Please try again.", "red")) - """ - elif choice == "7": - if os.path.exists(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.exists(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv") and os.path.exists(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv"): - allowpaths = utils.allowfunctions.filter_and_drop(pd.read_csv(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv"),tryToReadCSV(f"dataframe_csv\\df_path_eligible_{first_policy}_{second_policy}.csv"), path_exclusion_constant) - allowpaths.to_html(f"dataframe_html\\df_allowed_paths_{first_policy}_{second_policy}.html", index=False) - allowpaths.to_csv(f"dataframe_csv\\df_allowed_paths_{first_policy}_{second_policy}.csv", index=False) - print(ct.colorText(f"Allowable paths determined","green")) - else: - print(ct.colorText(f"Please complete step 6 prior to attempting this step","red")) - """ - + def tryToReadCSV(csv): try: