.gitignore

@@ -1,4 +1,6 @@
 .env
 *.html
 *.csv
-*__pycache__*
+*__pycache__*
+*.parquet
+chunkinator.json

AirlockTools.py

@@ -14,49 +14,69 @@
 #    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import dotenv
+import gc
+import json
 import os
-import utils.getdeviceevents
+import pandas as pd
+import urllib3
 import utils.allowlist
+import utils.getdeviceevents
 import utils.hashfunctions
 import utils.pathfunctions
+import utils.policyfunctions
 import utils.pretty as ct
-import urllib3
-import pandas as pd
-import ast
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
 dotenv.load_dotenv()
 
-url = "https://172.17.22.240:3129"
+#Constants
+url = os.getenv('url')
 badpublisherlist = ["Brave Software, Inc.",	"Zoom Video Communications, Inc."]
+badpathparts = ["users", "inet\\wwwroot", "windows\\temp", "windows\\temp", "windows\\task", "windows\\system32", "startup", "windows\\fonts", "Recycle.Bin", "AppData", "programdata"]
 path_exclusion_constant = 3
+min_files_for_path = 4
 threat_tolerance_constant = 4
 
 
-
 def apivalidation(): 
-	print(ct.colorText(r"""
-   _____  .__       .__                 __     ___________           .__          
-  /  _  \ |__|______|  |   ____   ____ |  | __ \__    ___/___   ____ |  |   ______
- /  /_\  \|  \_  __ \  |  /  _ \_/ ___\|  |/ /   |    | /  _ \ /  _ \|  |  /  ___/
-/    |    \  ||  | \/  |_(  <_> )  \___|    <    |    |(  <_> |  <_> )  |__\___ \ 
-\____|__  /__||__|  |____/\____/ \___  >__|_ \   |____| \____/ \____/|____/____  >
-        \/                           \/     \/                                 \/ 
-""", "cyan"))
-	print(ct.colorText("=================================================================================", "cyan"))
-	print(ct.colorText("======================== Welcome to the Airlock API Tool ========================", "cyan"))
-	print(ct.colorText("=================================================================================", "cyan"))
 	match os.getenv('APIKEY'):
 		case '':
 			print(ct.colorText("Please add your API Key to the .env file", "red"))
 		case _:
 			menu_main()
 
+def tryToReadCSV(csv):
+	try:
+		df =pd.read_csv(csv)
+		if df.empty:
+			print(ct.colorText("Error: CSV file has headers but no data rows.", "red"))
+		else:
+			print(ct.colorText(f"Data loaded successfully from {csv}", "green"))
+	except pd.errors.EmptyDataError:
+			print(ct.colorText("Notice : CSV file is completely empty (no headers, no data), falling back to empty frame", "white"))
+			df = pd.DataFrame()  # Create an empty DataFrame as fallback
+	return df
+
+def tryToReadParquet(parquet):
+	try:
+		df = pd.read_parquet(parquet)
+		if df.empty:
+			print(ct.colorText("Error: Parquet file has headers but no data rows.", "red"))
+		else:
+			print(ct.colorText(f"Data loaded successfully from {parquet}", "green"))
+	except pd.errors.EmptyDataError:
+			print(ct.colorText("Notice : Parquet file is completely empty (no headers, no data), falling back to empty frame", "white"))
+			df = pd.DataFrame()  # Create an empty DataFrame as fallback
+	return df
+
+def deduplicate_list(lst):
+    seen = set()
+    return [x for x in lst if not (x in seen or seen.add(x))]
+
 def menu_main():
 	while True:
-		print(ct.colorText("\n-----------------------------------", "magenta"))
+		ct.displayIntro();
-		print(ct.colorText("------------ Main Menu ------------", "magenta"))
-		print(ct.colorText("-----------------------------------", "magenta"))
 		print(ct.colorText("1. Get All Events for Single Device", "yellow"))
 		print(ct.colorText("2. Placeholder for Local Approval", "yellow"))
 		print(ct.colorText("3. Placeholder for Another Tool", "yellow"))
@@ -117,20 +137,25 @@ def menu_prepare_to_enforce():
 
 	first_policy = " "
 	second_policy = " "
-	
+	allowlist_parent_name = " "
+	allowlist_child_name = " "
+	destination_name = " "
+	df_aggregated_combo = pd.DataFrame()
 
 	#If the directorys where we're going to store our output dont exist, make them. 
-	if not os.path.exists("dataframe_html"): os.makedirs("dataframe_html")
+	if not os.path.exists("parquet"): os.makedirs("parquet")
-	if not os.path.exists("dataframe_csv"): os.makedirs("dataframe_csv")
+	if not os.path.exists("needs_approved"): os.makedirs("needs_approved")
-	if not os.path.exists("manuallyapproved"): os.makedirs("manuallyapproved")
+	if not os.path.exists("approved"): os.makedirs("approved")
+	if not os.path.exists("preflight"): os.makedirs("preflight")
 
-	df_aggregated_combo = pd.DataFrame()
 	while True:
 		print(ct.colorText("\n --------------------------------------------------------------------", "cyan"))
 		print(ct.colorText(" -------------------- Prepare to Enforce Policy ---------------------", "cyan"))
 		print(ct.colorText(" --------------------------------------------------------------------", "cyan"))
 		print(ct.colorText("\nSequentually follow these steps to prepare a policy for enforcement:", "white"))
-		print(ct.colorText("\n1. Choose which policy or policies to work with - : ", "cyan"))
+		
+		
+		print(ct.colorText("\n1. Choose which originating policy or policies to move to enforcement", "cyan"))
 		
 		if first_policy == " " and second_policy == " ": 
 			print(ct.colorText(f"  [✗]  No policies have been chosen","red"))
@@ -140,199 +165,473 @@ def menu_prepare_to_enforce():
 			print(ct.colorText(f"  [✓]  {first_policy} has been selected as Policy 1","green"))
 			print(ct.colorText(f"  [✓]  {second_policy} has been selected as Policy 2","green"))
 
-		print(ct.colorText("2. Pull and stage event history", "cyan"))
-		
-		if os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") == True: 
-			print(ct.colorText(f"  [✓]  This has been completed for {first_policy}","green"))
-		elif os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") == False: 
-			print(ct.colorText(f"  [✗]  This step has not been completed","red"))
-		elif second_policy is not first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv") == True:
-			print(ct.colorText(f"  [✓]  This has been completed for {second_policy}","green"))
-		elif second_policy is not first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv") == False:
-			print(ct.colorText(f"  [✓]  This has not been completed for {second_policy}","red"))
-
-		print(ct.colorText("3. Combine Staged policies", "cyan"))
-		
-		if os.path.exists(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv") == True:
-			print(ct.colorText("  [✓]  This step has been completed","green"))
-		else:
-			print(ct.colorText("  [✗]  This step has not been completed","red"))
-
-		print(ct.colorText("4. Add hash threat information to list of executions", "cyan"))
 	
-		if os.path.exists(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv") == True:
+		print(ct.colorText("2. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", "cyan"))
-			print(ct.colorText("  [✓]  This step has been completed","green"))
+		
-		else:
+		if os.path.exists(f"parquet\\execution_history_{first_policy}.parquet"): 
-			print(ct.colorText("  [✗]  This step has not been completed","red"))
+			print(ct.colorText(f"  [✓]  Execution history has been compiled for {first_policy}","green"))
+		elif not os.path.exists(f"parquet\\execution_history_{first_policy}.parquet"): 
+			print(ct.colorText(f"  [✗]  Execution history has not been compiled for {first_policy}","red"))
+		elif second_policy is not first_policy and os.path.exists(f"parquet\\execution_history_{second_policy}.parquet"):
+			print(ct.colorText(f"  [✓]  Execution history has been compiled for {second_policy}","green"))
+		elif second_policy is not first_policy and not os.path.exists(f"parquet\\execution_history_{second_policy}.parquet"):
+			print(ct.colorText(f"  [✗]  Execution history has not been compiled for {second_policy}","red"))
 
-		print(ct.colorText("5. Categorize your hashes ", "cyan"))
+		if 	os.path.exists(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"):
-				   
+			print(ct.colorText(f"  [✓]  Hash Info has been added to the combined execution history", "green"))
-		if os.path.isfile(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv"):
+		else: 
-			print(ct.colorText("  [✓]  This step has been completed","green"))
+			print(ct.colorText(f"  [✗]  Hash Info has not been added to the combined execution history", "red"))
-		else:
-			print(ct.colorText("  [✗]  This step has not been completed","red"))
 
-		print(ct.colorText(f"6. Manually review the files \\dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv and dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", "cyan"))
+		if os.path.exists(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet") and os.path.exists(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet") and os.path.exists(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet"):
+			print(ct.colorText(f"  [✓]  Hashes have been cateogrized", "green"))
+		else:
+			print(ct.colorText(f"  [✗]  Hashes have not been cateogrized", "red"))
+
+		if os.path.exists(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet"):
+			print(ct.colorText(f"  [✓]  Execution history has been_combined_for {first_policy} and_{second_policy}", "green"))
+		else: 
+			print(ct.colorText(f"  [✗]  Execution history has not been_combined_for {first_policy} and_{second_policy}", "red"))
+
+
+		
+		print(ct.colorText(f"3. Manually review the files:","cyan"))
+		print(ct.colorText("     'needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv' and 'needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv'", "cyan"))
 		print(ct.colorText("     Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.", "cyan"))
 		print(ct.colorText("     If metarules need to be created, please make note of them, and remove the row from the csv.", "cyan"))
-		print(ct.colorText("     When complete, save both csv files to the directory 'manuallyapproved' and choose this option to combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan"))
+		print(ct.colorText("     When complete, save both csv files to the directory 'approved' and choose this option.","cyan"))
+		print(ct.colorText("     This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan"))
 
-		if os.path.isfile(f"dataframe_csv\\df_paths_needing_review_{first_policy}_{second_policy}.csv"):
+		if os.path.exists(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv") and os.path.exists(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv"):
-			print(ct.colorText("  [✓]  This step has been completed","green"))
+			print(ct.colorText("  [✓]  Reviewed hashes have been loaded","green"))
 		else:
-			print(ct.colorText("  [✗]  This step has not been completed","red"))
+			print(ct.colorText("  [✗]  Reviewed hashes have not been loaded","red"))
 
-		print(ct.colorText(f"7. Manually review the file  df_paths_needing_review_{first_policy}_{second_policy}.csv", "cyan"))
+		if os.path.exists(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet"):
+			print(ct.colorText("  [✓]  The combined approved hashes list has been generated","green"))
+		else:
+			print(ct.colorText("  [✗]  The combined approved hashes list has not been generated","red"))
+
+		if os.path.exists(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet"):
+			print(ct.colorText("  [✓]  Longest common filepaths have been generated and appended to hash info","green"))
+		else:
+			print(ct.colorText("  [✗]  Longest common filepaths have not been generated","red"))
+
+		if os.path.exists(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
+				print(ct.colorText("  [✓]  Path review list created","green"))
+		else:
+			print(ct.colorText("  [✗]  Path review list has not been created","red"))	
+
+	
+		print(ct.colorText(f"4. Manually review the file 'needs_approved\\paths_needing_review_{first_policy}_{second_policy}.csv'", "cyan"))
 		print(ct.colorText("     Remove the rows containing path exclusions you do not approve of" , "cyan"))
-		print(ct.colorText("     When complete, save the csv file to the directory 'manuallyapproved' and choose this option to generate the proposed list of changes", "cyan"))	
+		print(ct.colorText("     When complete, save the csv file to the directory 'approved'", "cyan"))
+		print(ct.colorText("     Preflight Lists will be generated", "cyan"))
 
-		if os.path.isfile(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_hashdestination_{first_policy}_{second_policy}.csv"):
+		if  os.path.exists(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
-			print(ct.colorText("  [✓]  This step has been completed","green"))
+					print(ct.colorText("  [✓]  Reviewed path list detected","green"))
 		else:
-			print(ct.colorText("  [✗]  This step has not been completed","red"))
+			print(ct.colorText("  [✗]  Path review list has not been detected","red"))	
+	
+		if	os.path.exists(f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html"):
+			print(ct.colorText("  [✓]  Preflight Path Exclusion List has been generated","green"))
+		else:
+			print(ct.colorText("  [✗]  Preflight Path Exclusion List has not been generated","red"))	
+		
+		if os.path.exists(f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html"):
+			print(ct.colorText("  [✓]  Preflight hash approval list has been generated","green"))
+		else:
+			print(ct.colorText("  [✗]  Preflight hash approval list has not been generated","red"))	
 
 
+		print(ct.colorText(f"5. Choose the destination policy and parent and child allow list", "cyan"))	
+		if allowlist_child_name == " " and allowlist_parent_name== " ": 
+			print(ct.colorText(f"  [✗]  No allowlists have been chosen","red"))
+		elif allowlist_parent_name != " " and allowlist_child_name != " "  and allowlist_parent_name is allowlist_child_name:
+			print(ct.colorText(f"  [✓] [✗] Only {allowlist_parent_name} has been selected this is unusual, but potentially valid case, double check before proceeding,", "yellow"))
+		elif allowlist_parent_name != " " and allowlist_child_name != " "  and allowlist_parent_name is not allowlist_child_name:
+			print(ct.colorText(f"  [✓]  {allowlist_parent_name} has been selected as Parent Policy","green"))
+			print(ct.colorText(f"  [✓]  {allowlist_child_name} has been selected as Child Policy","green"))
+		if destination_name == " ":
+			print(ct.colorText(f"  [✗]  No destination policy has been chosen","red"))
+		else:
+			print(ct.colorText(f"  [✓]  destination policy is {destination_name}","green"))
+
+		print(ct.colorText(f"6. Liftoff ------------------------------------------------------", "cyan"))
+		print(ct.colorText(f"     Apply path exclusions according to allowed and approved paths", "cyan"))	
+		print(ct.colorText(f"     Apply signed or attested hashes to Parent Allow List", "cyan"))
+		print(ct.colorText(f"     Apply approved, but unsigned hashes to the Child Allow List", "cyan"))
+
 		print(ct.colorText("Q. Quit", "cyan"))
 
 		choice = input(ct.colorText("\nEnter your choice: ", "white"))
+		
 		if choice == "1":
-			first_policy_tuple = utils.allowlist.listPolicies(url)
+			
-			first_policy = first_policy_tuple[1][first_policy_tuple[0]]
+			choice, policynames, policyid = utils.allowlist.listPolicies(url)
+			first_policy = policynames[choice]
 			while True:
 				answer = input(ct.colorText(f"{"Do you want to load a second policy?"} (yes/no): ", "white").strip().lower())
 				if answer in ("yes", "y"):
-					second_policy_tuple = utils.allowlist.listPolicies(url)
+					choice, policynames, policyid = utils.allowlist.listPolicies(url)
-					second_policy = second_policy_tuple[1][second_policy_tuple[0]]
+					second_policy = policynames[choice]
+			
 					break 
 				elif answer in ("no", "n"):
-					second_policy_tuple = first_policy_tuple
 					second_policy = first_policy
 					break
 				else:
 					print(ct.colorText("Please answer with 'yes' or 'no'.", "red"))
 
 		elif choice == "2":
-			if  not os.path.exists("dataframe_csv\\df_aggregated_{first_policy}.csv"):
-				executionhist_policy1 = utils.allowlist.pullPolicyExechistories(url,first_policy_tuple[0], first_policy_tuple[1],True)
-				df_aggregated_policy1 = utils.hashfunctions.aggregateHashes(executionhist_policy1)
-				df_aggregated_policy1.to_csv(f"dataframe_csv\\df_aggregated_{first_policy}.csv", index=False)
-				ct.style_dataframe_dark(df_aggregated_policy1, f"dataframe_html\\df_aggregated_{first_policy}.html")
-				print(ct.colorText(f"Staging of Exection history for policy: {first_policy} is complete","green"))
-			
-			if not os.path.exists("dataframe_csv\\df_aggregated_{second_policy}.csv"):
-				executionhist_policy2 = utils.allowlist.pullPolicyExechistories(url,second_policy_tuple[0], second_policy_tuple[1],True)
-				df_aggregated_policy2 = utils.hashfunctions.aggregateHashes(executionhist_policy2)
-				df_aggregated_policy2.to_csv(f"dataframe_csv\\df_aggregated_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(df_aggregated_policy2, f"dataframe_html\\df_aggregated_{second_policy}.html")
-				print(ct.colorText(f"Staging of Exection history for policy: {second_policy} is complete","green"))
 
-		elif choice == "3":					
+			if not os.path.exists(f"parquet\\execution_history_{first_policy}.parquet"):
-			if  second_policy is first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv"):
+				print(choice)
-				df1 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{first_policy}.csv")
+				print(first_policy)
-				df_aggregated_combo = df1
-				df_aggregated_combo.to_csv(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\df_aggregated_combo_{first_policy}_{second_policy}.html")
-				print(ct.colorText(f"Dataframes have been aggregated (combined)","green"))
 
-			elif os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv"):
+				exe1 = utils.allowlist.pullPolicyExechistories(url, first_policy, 60, True)
-				df1 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{first_policy}.csv")
+				data = json.loads(exe1)
-				df2 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{second_policy}.csv")
+				executionhist_policy1 = pd.DataFrame(data["response"]["exechistories"])
-				df_aggregated_combo = pd.concat([df1 , df2], ignore_index=True)
-				df_aggregated_combo.to_csv(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\df_aggregated_combo_{first_policy}_{second_policy}.html")
-				print(ct.colorText(f"Dataframes have been aggregated (combined)","green"))
 
-			else:
+				if not executionhist_policy1.empty:
-				print(ct.colorText(f"Please stage your data before attempting this step","red"))
+					executionhist_policy1 = executionhist_policy1[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
+					executionhist_policy1 = executionhist_policy1.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
+					executionhist_policy1 = executionhist_policy1.sort_values(by=['sha256', 'filename'])
 
-		elif choice == "4":
+				executionhist_policy1.to_parquet(f"parquet\\execution_history_{first_policy}.parquet", index=False)
-			if os.path.exists(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv"):
+				print(ct.colorText(f"Staging of Execution history for policy: {first_policy} is complete", "green"))
-				df_augmented = utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv"))
-				df_augmented.to_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(df_augmented, f"dataframe_html\\df_augmented_combo_{first_policy}_{second_policy}.html")
-				print(ct.colorText(f"Hash reputation info added to dataframe","green"))
-			
-			else:
-				print(ct.colorText(f"Please combine your data with step 3 prior to attempting this step","red"))
-
-		elif choice == "5":
-			if os.path.exists(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv"):
-				categorized = utils.hashfunctions.categorizeHashes(pd.read_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv"), threat_tolerance_constant, badpublisherlist)
-
-				categorized[0].to_csv(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(categorized[0], f"dataframe_html\\df_hashes_needing_approval_{first_policy}_{second_policy}.html")
-
-				categorized[1].to_csv(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(categorized[1], f"dataframe_html\\df_automatically_approved_hashes_{first_policy}_{second_policy}.html")
-
-				categorized[2].to_csv(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(categorized[2], f"dataframe_html\\df_unapproved_hashes_{first_policy}_{second_policy}.html")
-
-				print(ct.colorText(f"Hashes have been categorized","green"))
-
-			else:
-				print(ct.colorText(f"Please Augment your data with hash threat info using step 4 prior to attempting this step","red"))
-
-		elif choice == "6":
-			if os.path.exists(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.exists(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv"):
-				df1 = tryToReadCSV(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv")
-				df2 = tryToReadCSV(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv")
-
-				df_all_approved_hashes = pd.concat([df1 , df2], ignore_index=True)
-				df_all_approved_hashes.to_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
-				ct.style_dataframe_dark(df_all_approved_hashes, f"dataframe_html\\df_all_approved_hashes_{first_policy}_{second_policy}.html")
 				
-				df_paths_needing_review, df_path_ineligible = utils.pathfunctions.filepathInitialGroup(pd.read_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv"))
+				del data
+				del exe1
+				del executionhist_policy1
+				
+				gc.collect()
 
-				df_paths_needing_review.to_csv(f"dataframe_csv\\df_paths_needing_review_{first_policy}_{second_policy}.csv", index=False)
+			if not os.path.exists(f"parquet\\execution_history_{second_policy}.parquet"):
-				ct.style_dataframe_dark(df_paths_needing_review, f"dataframe_html\\df_paths_needing_review_{first_policy}_{second_policy}.html")
+				exe2 = utils.allowlist.pullPolicyExechistories(url,second_policy, 60, True)
+				data2 = json.loads(exe2)
+				executionhist_policy2 = pd.DataFrame(data2["response"]["exechistories"])
 
-				df_path_ineligible.to_csv(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv", index=False)
+				if not executionhist_policy2.empty:
-				ct.style_dataframe_dark(df_path_ineligible, f"dataframe_html\\df_path_ineligible_{first_policy}_{second_policy}.html")
+					executionhist_policy2 = executionhist_policy2[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
+					executionhist_policy2 = executionhist_policy2.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
+					executionhist_policy2 = executionhist_policy2.sort_values(by=['sha256', 'filename'])
 
-				print(ct.colorText(f"Eligible paths determined","green"))
+				executionhist_policy2.to_parquet(f"parquet\\execution_history_{second_policy}.parquet", index=False)
+				print(ct.colorText(f"Staging of Execution history for policy: {second_policy} is complete", "green"))
+
+				del executionhist_policy2
+				del data2
+				del exe2
+
+				gc.collect()
+
+			if not os.path.exists(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"):
+				combined_hashes = pd.DataFrame(columns=['sha256', 'publisher'])
+				hashes = []
+
+				try:
+					hash1 = pd.read_parquet(f"parquet\\execution_history_{first_policy}.parquet", columns=['sha256', 'publisher'])
+					utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{first_policy}.parquet")
+					if not hash1.empty:
+						hashes.append(hash1)
+					else:
+						print("⚠️ First dataframe is empty.")
+				except Exception as e:
+					print(f"❌ Error reading first Parquet file: {e}")
+
+				try:
+					hash2 = pd.read_parquet(f"parquet\\execution_history_{second_policy}.parquet", columns=['sha256', 'publisher'])
+					utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{second_policy}.parquet")
+					if not hash2.empty:
+						hashes.append(hash2)
+					else:
+						print("⚠️ Second dataframe is empty.")
+				except Exception as e:
+					print(f"❌ Error reading second Parquet file: {e}")
+
+				if hashes:
+					combined_hashes = pd.concat(hashes, ignore_index=True)
+					print(f"✅ Combined {len(combined_hashes)} hashes.")
+				else:
+					print("⚠️ No valid dataframes to combine.")
+
+				combined_hashes = combined_hashes.drop_duplicates(subset=['sha256'])
+				augmented_combo = utils.hashfunctions.augmentAggregatedHashes(url, combined_hashes)
+
+				numeric_reputation_cols = [
+					'reputation_scannermatch',
+					'reputation_scannercount',
+					'reputation_threatlevel'
+				]
+
+				for col in numeric_reputation_cols:
+					if col in augmented_combo.columns:
+						augmented_combo[col] = pd.to_numeric(augmented_combo[col].replace('N/A', pd.NA), errors='coerce')
+
+				augmented_combo = augmented_combo.rename(columns={'publisher_x': 'publisher'})
+				augmented_combo = augmented_combo[['sha256', 'publisher', 'description', 'productname', 'productversion',
+												'reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount',
+												'reputation_status', 'reputation_threatlevel', 'reputation_threatname',
+												'reputation_timestamp']]
+				augmented_combo = augmented_combo.sort_values(by=['publisher', 'description', 'productname'])
+				augmented_combo.to_parquet(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet", index=False)
+
+				del combined_hashes
+				del augmented_combo
+				gc.collect()
+
+				print(ct.colorText("Hash reputation info added to dataframe", "green"))
+
+			if not os.path.exists(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet"):
+
+				# Categorize the hashes
+				categorized = utils.hashfunctions.categorizeHashes(
+					pd.read_parquet(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"),
+					threat_tolerance_constant,
+					badpublisherlist
+				)
+
+				categorized[0].to_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet", index=False)
+				categorized[1].to_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet", index=False)
+				categorized[2].to_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet", index=False)
+
+				del categorized
+				gc.collect()
+			
+			if not os.path.exists(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet"):
+			# Condense execution history
+				try:
+					exe1 = pd.read_parquet(f"parquet\\execution_history_{first_policy}.parquet")
+					utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{first_policy}.parquet")
+					if not exe1.empty:
+						condensed_exe1 = exe1.groupby('sha256').agg(lambda x: list(set(x))).reset_index()
+					else:
+						print("⚠️ First dataframe is empty.")
+				except Exception as e:
+					print(f"❌ Error reading first Parquet file: {e}")
+
+				try:
+					exe2 = pd.read_parquet(f"parquet\\execution_history_{second_policy}.parquet")
+					utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{second_policy}.parquet")
+					if not exe2.empty:
+						condensed_exe2 = exe2.groupby('sha256').agg(lambda x: list(set(x))).reset_index()
+					else:
+						print("⚠️ Second dataframe is empty.")
+				except Exception as e:
+					print(f"❌ Error reading second Parquet file: {e}")
+
+				if not exe1.empty and not exe2.empty:
+					condensed_combo = pd.concat([condensed_exe1, condensed_exe2], ignore_index=True)
+					print(f"✅ Combined {len(condensed_combo)} hashes.")
+				elif exe1.empty:
+					condensed_combo = condensed_exe2
+				elif exe2.empty:
+					condensed_combo = condensed_exe1
+				else:
+					print("⚠️ No valid dataframes to combine.")
+
+				condensed_combo.to_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet", index=False)
+				del condensed_combo
+				gc.collect()
+
+			if not os.path.exists(f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv"):
+				
+				condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
+				needsapproval= pd.read_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet")
+				
+				#Pull hash info for the entries in the needs approval table
+				needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
+				
+				#Deduplicate lists in the columns
+				for col in needsapproval.columns:
+					if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
+						needsapproval[col] = needsapproval[col].apply(deduplicate_list)
+
+				#Rename Publisher, Keep and reorder columns we want
+				needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
+				needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
+				
+				needsapproval.to_csv(f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv",index=False)
+				needsapproval.to_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet",index=False)
+				ct.style_dataframe_dark(needsapproval, f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.html")
+
+
+				del needsapproval
+				del condensed_combo
+				gc.collect()
+
+			if not os.path.exists(f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv"):
+				
+				condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
+				needsapproval= pd.read_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet")
+				
+				#Pull hash info for the entries in the needs approval table
+				needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
+				
+				#Deduplicate lists in the columns
+				for col in needsapproval.columns:
+					if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
+						needsapproval[col] = needsapproval[col].apply(deduplicate_list)
+
+				#Rename Publisher, Keep and reorder columns we want
+				needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
+				needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
+				
+				needsapproval.to_csv(f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv",index=False)
+				needsapproval.to_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet", index=False)
+				ct.style_dataframe_dark(needsapproval, f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.html")
+
+
+				del needsapproval
+				del condensed_combo
+				gc.collect()
+				
+			if not os.path.exists(f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.html"):
+				
+				condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
+				needsapproval= pd.read_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet")
+				
+				#Pull hash info for the entries in the needs approval table
+				needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
+				
+				#Deduplicate lists in the columns
+				for col in needsapproval.columns:
+					if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
+						needsapproval[col] = needsapproval[col].apply(deduplicate_list)
+
+				#Rename Publisher, Keep and reorder columns we want
+				needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
+				needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
+				
+				needsapproval.to_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet", index=False)
+				ct.style_dataframe_dark(needsapproval, f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.html")
+				
+				del needsapproval
+				del condensed_combo
+				gc.collect()
+				
+		elif choice == "3":
+			
+			if os.path.exists(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv") and os.path.exists(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv"):
+			
+				if not os.path.exists(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet"):
+
+					df1 = tryToReadCSV(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv")
+					df2 = tryToReadCSV(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv")
+
+					all_approved_hashes = pd.concat([df1 , df2], ignore_index=True).sort_values(by=['sha256','filename'])
+					print(ct.colorText(f"Approved hash lists have been combined","green"))
+
+					all_approved_hashes.to_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet", index=False)
+					del all_approved_hashes
+					gc.collect()
+			
+				if not os.path.exists(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet"):
+					all_approved_hashes = pd.read_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet")
+					print(ct.colorText(f"Beginning calculating longest common filepaths for path exceptions","green"))
+					grouped_df_view, df_with_groups_appended = utils.pathfunctions.export_groups_for_review(all_approved_hashes,"filename","longestcfp",min_files_for_path,path_exclusion_constant)
+				
+					df_with_groups_appended.to_parquet(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet", index=False)
+		
+					forbidden = utils.pathfunctions.regulator(badpathparts, True)
+					forbidden_lcfp = grouped_df_view["longestcfp"].str.contains(forbidden, na=False)
+					grouped_df_view = grouped_df_view[~forbidden_lcfp]
+					print(ct.colorText(f"Removing forbidden filepaths for path exceptions","green"))
+
+					grouped_df_view.to_parquet(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet", index=False)
+					grouped_df_view.to_csv(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv", index=False)
+					ct.style_dataframe_dark(grouped_df_view, f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.html")
+					
+					del grouped_df_view
+					del df_with_groups_appended
+					gc.collect()
 
 			else:
 				print(ct.colorText(f"Please manually approve hashes prior to this step","red"))
 
-		
+
-		elif choice == "7":
+		elif choice == "4":
-			if os.path.exists(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv"):
-				df1 = tryToReadCSV(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv")
-				df2 = tryToReadCSV(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv")
-				df3 = tryToReadCSV(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv")
-				df_hashdestination = utils.hashfunctions.destinationbuilder(df2,df3)
-				df_hashdestination.to_csv("dataframe_csv\\df_hashdestination_{first_policy}_{second_policy}.csv")
-				ct.style_dataframe_dark(df_hashdestination,f"dataframe_html\\df_hashdestination_{first_policy}_{second_policy}.html")
-				
-			else:
-				print(ct.colorText(f"Please manually approve suggested paths prior to this step","red"))
 			
+			if os.path.exists(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet") and os.path.exists(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
+				if not os.path.exists(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet"):
+					df1 = pd.read_parquet(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet")
+					allowbyhash = utils.pathfunctions.mask_from_csv(df1, f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv","longestcfp")
+					
+					pathexclusions = tryToReadCSV(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv")
+					
+					pathexclusions.to_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet", index=False)
+		
+					allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False)
+				
+					easyview = allowbyhash.groupby('sha256').agg(list).reset_index()
+			
+					ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
+					ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html")
+
+					del df1
+					del allowbyhash
+					del pathexclusions
+					del easyview
+					gc.collect()
 
 
 
+		elif choice == "5":
+	
+			print(ct.colorText(f"Please choose destination_name Policy for Path Exclusions","white"))
+			choice, policynames, policyid = utils.allowlist.listPolicies(url)
+			#print(allowlist_parent_tuple)
+			destination_name = policynames[choice]
+			destination_id = policyid[choice]
+			
+			print(ct.colorText(f"Please choose Parent Allowlist for Known Hashes","white"))
+			choice, allowlists,allowid = utils.allowlist.listAllowlists(url)
+			#print(allowlist_parent_tuple)
+			allowlist_parent_name = allowlists[choice]
+			allowlist_parent_id = allowid[choice]
 
+			print(ct.colorText(f"Please choose Child Allowlist for Less-Known Hashes","white"))
+			choice, allowlists, allowid = utils.allowlist.listAllowlists(url)
+			#print(allowlist_child_tuple)
+			allowlist_child_name = allowlists[choice]
+			allowlist_child_id = allowid[choice]
+
+		elif choice == "6":
+			if os.path.exists(f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html") and os.path.exists(f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")	and allowlist_parent_name != " " and allowlist_child_name != " " and destination_name != " ":
+
+				pathexclusions = pd.read_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet")
+				allowbyhash =  pd.read_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet")
+
+				ct.areYouSure()
+				confirmation = input(ct.colorText("Type 'I AGREE' to continue: ","white"))
+
+				if confirmation.strip().upper() == "I AGREE":
+					print(ct.colorText("Proceeding with the code...", "yellow"))
+					print(ct.colorText(f"Adding path exclusions to {destination_name}", "yellow"))
+					pathexcludelist = pathexclusions['longestcfp'].unique().tolist()
+					utils.policyfunctions.addPath(destination_id,pathexcludelist)
+					
+					print(ct.colorText(f"Adding hashes to {allowlist_parent_name}", "yellow"))
+										
+					allowlist_parenthashlist = allowbyhash[allowbyhash['reputation_status'] == 'KNOWN']['sha256'].unique().tolist()
+					utils.policyfunctions.addHash(allowlist_parent_id,allowlist_parenthashlist)
+					
+					print(ct.colorText(f"Adding hashes to {allowlist_child_name}", "yellow"))
+					allowlist_childhashlist = allowbyhash[allowbyhash['reputation_status'] == 'UNKNOWN']['sha256'].unique().tolist()
+					utils.policyfunctions.addHash(allowlist_child_id, allowlist_childhashlist)			
+
+					ct.locked()
+					exit()		
+					
+				else:
+					print(ct.colorText("Operation aborted. You MUST EXPLICITLY AGREE to proceed.", "red"))
+					break
 
 		elif choice == "Q":
 			break
 		else:
 			print(ct.colorText("Invalid choice. Please try again.", "red"))    
 		
-		
-
-def tryToReadCSV(csv):
-	try:
-		df =pd.read_csv(csv)
-		if df.empty:
-			print(ct.colorText("Error: CSV file has headers but no data rows.", "red"))
-		else:
-			print(ct.colorText("Data loaded successfully.", "green"))
-	except pd.errors.EmptyDataError:
-			print(ct.colorText("Notice : CSV file is completely empty (no headers, no data), falling back to empty frame", "white"))
-			df = pd.DataFrame()  # Create an empty DataFrame as fallback
-	return df
-
-
 
 if __name__ == "__main__":
 	apivalidation()

allowlistandgroup.py

@@ -0,0 +1,22 @@
+import requests
+import dotenv
+import json
+import os
+import utils.pretty as ct
+
+url = 'https://172.17.22.240:3129'
+policiesnames = []
+policyids=[]
+dotenv.load_dotenv()
+endpoint = url + '/v1/application'
+print(ct.colorText("[+] Grabbing All Allowlists", "cyan"))
+payload = {}
+headers = {
+    "X-APIKey": os.getenv('APIKEY')
+}
+response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+parse_text = json.loads(response.text)
+for index, list in enumerate(parse_text['response']['applications'], start=1):
+        if index >= 38:
+            print(list)
+#Need else and catch for upper bound

utils/allowfunctions.py

@@ -1,16 +0,0 @@
-import pandas as pd
-
-def filter_and_drop(approved, eligiblepaths, min_hashes):
-    """
-    Filters eligiblepaths to rows where all hashes are in approved,
-    then drops rows with fewer than min_hashes hashes.
-    """
-    approved_hashes = set(approved['sha256'])
-
-    def all_hashes_approved(row):
-        return all(h in approved_hashes for h in row['sha256'])
-
-    filtered = eligiblepaths[eligiblepaths.apply(all_hashes_approved, axis=1)]
-    filtered = filtered[filtered['sha256'].apply(len) >= min_hashes]
-    
-    return filtered

utils/allowlist.py

@@ -16,48 +16,67 @@ import datetime
 import requests
 import json
 import os
-import time
 import utils.pretty as ct
 import ijson
+import os
+from bson import ObjectId
+import datetime
 
-def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): 
+def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
-    headers = {
+    file_path = 'chunkinator.json'
-        "X-APIKey": os.getenv('APIKEY')
+    if not os.path.exists(file_path):
-    }
+        with open(file_path, 'w') as file:
-    checkpoint = '000000000000000000000000'
+            json.dump({'error': 'Success', 'response': {'exechistories': []}}, file)
+        print(f"File '{file_path}' has been crated.")
+    else:
+        print(f"File '{file_path}' already exists.")
+    headers = {"X-APIKey": os.getenv('APIKEY')}
+    checkpoint = str(skipback(days))
     json_output = {'error': 'Success', 'response': {'exechistories': []}}
     while True:
-        json_response_data = checkpoint_stomper(checkpoint, url, policiesnames[choice], headers)
+        json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers)
-        if not json_response_data['response']['exechistories']:
+        histories = json_response_data['response']['exechistories']
+        if not histories:
             break
+        array_dividend =  max(round(len(histories) / 20), 1)
         match_found = False
-        array_dividend = round(len(json_response_data['response']['exechistories'])/20)
+        for index, item in enumerate(histories[::array_dividend]):
-        if array_dividend == 0:
+            if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
-            array_dividend == 1
-        for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]):
-            if (datetime.date.today() - datetime.timedelta(days=30) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
-                print("Found Date Match")
                 match_found = True
                 break
-        checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend)
+        checkpoints_processed = round(len(histories) / array_dividend)
-        print(ct.colorText(f"{checkpoints_processed} checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue"))
+        if ( index + 1 ) < checkpoints_processed:
-        checkpoint = item['checkpoint']
+            print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue"))
+        else:
+            print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue"))
         if match_found == True:
-            for index, item in enumerate(json_response_data['response']['exechistories']):
+            for index, item in enumerate(histories):
-                if index == len(json_response_data['response']['exechistories']) -1:
+                if index == len(histories) - 1:
+                    print(ct.colorText(f"All Events Processed for {checkpoint}", "blue"))
                     checkpoint = item['checkpoint']
-                    print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue"))
                     break
                 else:
-                    if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
+                    if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
                         pass
-                    else:
+                    else: json_output['response']['exechistories'].append(item)
-                        print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}")
+        seen = {}
-                        json_output['response']['exechistories'].append(item)
+        if os.path.exists(file_path):
-            match_found = False
+            with open(file_path, 'r') as file:
-    json_output = json.dumps(json_output)
+                existing_data = json.load(file)
-    if outputjson == True:
+                combined = existing_data['response']['exechistories']  + json_output['response']['exechistories']
-        return json_output
+        else:
+            combined = json_output['response']['exechistories']
+        for item in combined:
+            key = (item.get('sha256'), item.get('filename'), item.get('hostname'))
+            seen[key] = item
+        deduplicated = list(seen.values())
+        with open(file_path, 'w') as file:
+            json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file)
+        json_output['response']['exechistories'].clear()
+    with open(file_path, 'r') as file:
+        final_output = json.load(file)
+    os.remove(file_path)
+    return json.dumps(final_output) if outputjson else None
 
 def checkpoint_stomper(checkpoint, url, policy, headers):
     json_output = {'error': 'Success', 'response': {'exechistories': []}}
@@ -94,4 +113,40 @@ def listPolicies(url):
         policyids.append(list['groupid'])
     choice = input(ct.colorText("Select Policy Group: ", "white"))
     choice = int(choice) - 1
-    return choice, policiesnames
+    return choice, policiesnames, policyids
+
+def listAllowlists(url): 
+    endpoint = url + '/v1/application'
+    print(ct.colorText("[+] Grabbing All Allowlists", "cyan"))
+    payload = {}
+    headers = {
+        "X-APIKey": os.getenv('APIKEY')
+    }
+    response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+    parse_text = json.loads(response.text)
+    policiesnames = []
+    policyids = []
+    for index, list in enumerate(parse_text['response']['applications'], start=1):
+        if index >= 38:
+            print(ct.colorText(f"{index}. {list['name']}", "yellow"))
+            policiesnames.append(list['name'])
+            policyids.append(list['applicationid'])
+    choice = int(input(ct.colorText("Select allowlist: ", "white")))
+    if choice < 38:
+        print(ct.colorText("Please only choose an allowlist designed for this use - '38+'","red"))
+    elif choice >= 38:
+        choice = choice - 38
+        return choice, policiesnames, policyids
+    #Need else and catch for upper bound
+
+def skipback(days):
+    """
+    Generate a MongoDB ObjectId for a given number of days ago from today.
+    Adds 1 extra day to the input to look further back.
+    """
+    adjusted_days = days + 1
+    date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days)
+    timestamp = int(date_days_ago.timestamp())
+    hex_timestamp = format(timestamp, '08x')
+    objectid_hex = hex_timestamp + '0000000000000000'
+    return ObjectId(objectid_hex)

utils/hashfunctions.py

@@ -12,12 +12,12 @@
 #
 #    You should have received a copy of the GNU Affero General Public License
 #    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import pandas as pd
 import requests
 import os
 import json
-
+import utils.pretty as ct
-
 
 def aggregateHashes(executions_json) -> pd.DataFrame:
     """
@@ -45,18 +45,22 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
     Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash,
     aggregate applications and baselines into lists, then merges results back into agg_df to create a 
     """
+    if 'sha256' not in agg_df.columns or agg_df.empty:
+        print("⚠️ 'sha256' column missing or DataFrame is empty. Skipping API query.")
+        return agg_df.copy()  # Return as-is to avoid breaking downstream logic
+
     endpoint = url + '/v1/hash/query'
     payload = {
-            "hashes": agg_df['sha256'].tolist()
+        "hashes": agg_df['sha256'].tolist()
     }
-    
+
     headers = {"X-APIKey": os.getenv('APIKEY')}
     payload = json.dumps(payload)
-  
+
     response = requests.post(endpoint, headers=headers, data=payload, verify=False)
-    data = response.json()  
+    data = response.json()
     results = data.get("response", {}).get("results", [])
-    
+
     rows = []
     for res in results:
         row = {"sha256": res.get("sha256"), "result": res.get("result")}
@@ -79,10 +83,23 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
 
     df_api = pd.DataFrame(rows)
 
+    if 'sha256' not in df_api.columns:
+        print("⚠️ API response missing 'sha256'. Skipping merge.")
+        return agg_df.copy()
+
     df = agg_df.merge(df_api, on="sha256", how="left")
-    aug_df = df[['sha256', 'filename_x', 'description', 'productname', 'productversion', 'publisher_y', 'publisher_x', 'netdomain', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline', 'reputation_lastseen', 'reputation_scannercount', 'reputation_scannermatch', 'reputation_status', 'reputation_threatlevel', 'reputation_threatname', 'reputation_timestamp']]
+
+    # Only include columns that exist to avoid KeyErrors
+    expected_columns = ['sha256', 'filename_x', 'description', 'productname', 'productversion',
+                        'publisher_y', 'publisher_x', 'netdomain', 'hostname', 'username',
+                        'reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount',
+                        'reputation_status', 'reputation_threatlevel', 'reputation_threatname',
+                        'reputation_timestamp', 'pprocess', 'gprocess', 'commandline']
+
+    available_columns = [col for col in expected_columns if col in df.columns]
+    aug_df = df[available_columns]
+
     return aug_df
- 
 
 def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
     if untrusted_publishers is None:
@@ -93,29 +110,29 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
     def reputationtool(row):
         val = row["reputation_scannermatch"]
         if pd.isna(val) or val == "N/A":
-            return row["publisher_y"] == "Not Signed"
+            return row["publisher"] == "Not Signed"
         try:
             return int(val) > threat_tolerance
         except (ValueError, TypeError):
-            return row["publisher_y"] == "Not Signed"
+            return row["publisher"] == "Not Signed"
 
     df["reputation_flag"] = df.apply(reputationtool, axis=1)
 
     mask_needsreview = (
-        ((df["publisher_y"] == "Not Signed") & df["reputation_flag"]) |
+        ((df["publisher"] == "Not Signed") & df["reputation_flag"]) |
         (df["reputation_status"] == "UNKNOWN")
     )
 
     mask_approved = (
         (
-            (df["publisher_y"] != "Not Signed") &
+            (df["publisher"] != "Not Signed") &
-            ~df["publisher_y"].isin(untrusted_publishers) &
+            ~df["publisher"].isin(untrusted_publishers) &
             ~df["reputation_status"].isna()
         ) |
         (
-            (df["publisher_y"] == "Not Signed") &
+            (df["publisher"] == "Not Signed") &
             ~df["reputation_flag"] &
-            ~df["publisher_y"].isin(untrusted_publishers) &
+            ~df["publisher"].isin(untrusted_publishers) &
             ~df["reputation_status"].isna()
         )
     )
@@ -126,18 +143,59 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
 
     return needsreview_df, approved_df, unapproved_df
 
-def destinationbuilder(df, df2):
+def explode_and_deduplicate(df):
-    # Step 1: Explode the 'sha256' list in df2 to create one row per sha256 value
+    df['sha256'] = df['sha256'].str.split(',')
-    df_expanded = df.explode('sha256')
+    df = df.explode('sha256')
+    return df.drop_duplicates().reset_index(drop=True)
 
-    # Step 2: Create a new dataframe for the result
+def clean_sha256(df, column='sha256'):
-    df_hashdestination = df_expanded.copy()
+    """Discard quotes, brackets, and whitespace from sha256 values."""
+    df[column] = df[column].astype(str).str.strip("'[]\" ")
+    return df
 
-    # Step 3: Populate the 'Destination Allowlist' column based on comparison with df3
+def destinationHashes(
-    df_hashdestination['Destination Allowlist'] = df_hashdestination['sha256'].apply(
+    df_approved_paths: pd.DataFrame,
-        lambda x: 'Parent Policy Baseline' if x in df2['sha256'].values else "Destination Policy Allowlist"
+    df_approved_hashes: pd.DataFrame,
+    df_hashes_auto_approved: pd.DataFrame,
+    df_hashes_manually_approved: pd.DataFrame,
+):
+    # Deduplicate and explode all input DataFrames
+    df_approved_paths = explode_and_deduplicate(df_approved_paths)
+    df_approved_hashes = explode_and_deduplicate(df_approved_hashes)
+    df_hashes_auto_approved = explode_and_deduplicate(df_hashes_auto_approved)
+    df_hashes_manually_approved = explode_and_deduplicate(df_hashes_manually_approved)
+
+    # Clean sha256 values in all relevant DataFrames
+    df_approved_hashes = clean_sha256(df_approved_hashes)
+    df_hashes_auto_approved = clean_sha256(df_hashes_auto_approved)
+    df_hashes_manually_approved = clean_sha256(df_hashes_manually_approved)
+
+    # Create sets for faster lookup
+    auto_approved_sha256 = set(df_hashes_auto_approved['sha256'].values)
+    manually_approved_sha256 = set(df_hashes_manually_approved['sha256'].values)
+
+    # Debug: Print unmatched hashes
+    unmatched = set(df_approved_hashes['sha256']) - (auto_approved_sha256 | manually_approved_sha256)
+    print(f"Unmatched hashes: {unmatched}")
+
+    # Process df_approved_paths
+    df_paths = df_approved_paths.assign(destination='Path Exclusion')
+    df_paths = df_paths[['sha256', 'description', 'destination', 'grouped_directory', 'filename']]
+
+    # Process df_approved_hashes
+    df_hashes = df_approved_hashes.copy()
+    df_hashes['destination'] = df_hashes['sha256'].apply(
+        lambda x: 'Parent Policy Baseline' if x in auto_approved_sha256
+                  else ('Child Policy Allowlist' if x in manually_approved_sha256 else None)
     )
+    df_hashes = df_hashes.dropna(subset=['destination'])
+    df_hashes = df_hashes.assign(grouped_directory=None)
 
-    # Step 4: Return the new dataframe
+    # Use 'filename_x' only if it exists, otherwise fallback to 'filename'
+    filename_col = 'filename_x' if 'filename_x' in df_hashes.columns else 'filename'
+    selected_cols = ['sha256', 'description', 'destination', 'grouped_directory', filename_col]
+    df_hashes = df_hashes[selected_cols]
+
+    # Concatenate results
+    df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
     return df_hashdestination
-

utils/pathfunctions.py

@@ -16,103 +16,137 @@
 import pandas as pd
 import os
 from itertools import chain
+import ast
+import re
 
-def filepathInitialGroup(df: pd.DataFrame):
+def split_path(path):
-    original_columns = df.columns.tolist()
+    parts = []
+    while True:
+        head, tail = os.path.split(path)
+        if tail:
+            parts.insert(0, tail)
+            path = head
+        else:
+            if head:
+                parts.insert(0, head)
+            break
+    return parts
 
-    # Step 1: Split comma-separated filepaths into lists
+def local_common_pass(paths, min_parts=3):
-    df["filename_x"] = df["filename_x"].str.split(",")
+    results = {}
+    paths_sorted = sorted(paths)
+    for i, path in enumerate(paths_sorted):
+        candidates = []
 
-    # Step 2: Explode the list so each filepath becomes its own row
+        if i > 0:
-    df = df.explode("filename_x", ignore_index=True)
+            try:
+                candidates.append(os.path.commonpath([path, paths_sorted[i-1]]))
+            except ValueError:
+                # different drives, skip
+                pass
+        if i < len(paths_sorted) - 1:
+            try:
+                candidates.append(os.path.commonpath([path, paths_sorted[i+1]]))
+            except ValueError:
+                # different drives, skip
+                pass
 
-    # Step 3: Clean up whitespace and normalize paths
+        best = path
-    df["filename_x"] = df["filename_x"].str.strip()
+        best_len = 0
-    df["filename_x"] = df["filename_x"].str.replace(r"\\\\", r"\\", regex=True)
+        for c in candidates:
-    df["filename_x"] = df["filename_x"].apply(lambda x: os.path.normpath(x) if pd.notna(x) else "")
+            parts = split_path(c)
+            if len(parts) >= min_parts and len(parts) > best_len:
+                best = c
+                best_len = len(parts)
+        results[path] = best
+    return results
 
-    # Step 4: Extract directory and filename from each filepath
+def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3):
-    df["directory"] = df["filename_x"].apply(lambda x: os.path.normpath(os.path.dirname(x)) if pd.notna(x) else "")
+    dirs_series = df[col].astype(str).apply(os.path.dirname)
-    df["filename"] = df["filename_x"].apply(lambda x: os.path.basename(x) if pd.notna(x) else "")
+    first_pass = local_common_pass(dirs_series.tolist(), min_parts)
+    second_pass = local_common_pass(list(first_pass.values()), min_parts)
+    df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]])
+    return df
 
-    # Step 5: Drop the original raw filepath column
+def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant):
-    df = df.drop(columns=["filename_x"])
+    """
+    Compute longest common paths, group filepaths, write CSV for review.
+    """
+    df = df.drop_duplicates(subset=[col], keep='first')
+    df = add_longest_common_two_local(df, col=col, new_col=group_col)
+    grouped = df.groupby(group_col)[col].apply(list).reset_index()
+    grouped = grouped.sort_values(by=col)
+    print("Before filtering:", len(grouped))
+    grouped = grouped[grouped[col].apply(lambda x: len(x) >= min_number_in_group)]
+    filtered = grouped[grouped[group_col].apply(lambda x: len(os.path.normpath(x).split(os.sep)) >= path_length_constant)]
+    print("After filtering:", len(grouped))
 
-    # Helper functions for path manipulation
+    return filtered, df
-    def get_parts(path):
-        return os.path.normpath(path).split(os.sep)
 
-    def join_parts(parts):
+def mask_from_csv(df, csv_path, filepath_col):
-        return os.path.normpath(os.sep.join(parts))
+    """
+    Reads reviewed CSV of groups, keeps only files in approved groups.
+    """
+    review_df = pd.read_csv(csv_path)
 
-    def longest_common_prefix(paths):
+    def parse_paths(val):
-        split_paths = [get_parts(p) for p in paths]
+        if isinstance(val, str):
-        min_len = min(len(p) for p in split_paths)
+            try:
-        prefix = []
+                # Try to parse as a list
-        for i in range(min_len):
+                parsed = ast.literal_eval(val)
-            current = split_paths[0][i]
+                # If it's not a list, wrap it
-            if all(p[i] == current for p in split_paths):
+                return parsed if isinstance(parsed, list) else [parsed]
-                prefix.append(current)
+            except (ValueError, SyntaxError):
-            else:
+                # If parsing fails, treat it as a single path
-                break
+                return [val]
-        return join_parts(prefix)
+        return [val]
 
-    # Step 6: Group directories by shared prefix
+    review_df[filepath_col] = review_df[filepath_col].apply(parse_paths)
-    directories = df["directory"].tolist()
-    groups = []
-    used = set()
 
-    for i, path in enumerate(directories):
+    # Flatten all approved file paths into a set for masking
-        if path in used:
+    approved_files = set()
-            continue
+    for paths in review_df[filepath_col]:
-        group = [path]
+        approved_files.update(paths)
-        parts_i = get_parts(path)
 
-        for j in range(i + 1, len(directories)):
+    # Keep only rows in df that are in approved_files
-            parts_j = get_parts(directories[j])
+    masked_df = df[df[filepath_col].isin(approved_files)].copy()
-            common = os.path.commonprefix([parts_i, parts_j])
+    remainder = df[~df[filepath_col].isin(approved_files)].copy()
+    return remainder
 
-            if (len(parts_i) > 3 and len(common) >= 3) or (len(parts_i) == 3 and len(common) >= 2):
+def filter_and_drop(approved, eligiblepaths, min_hashes):
-                group.append(directories[j])
+    """
-                used.add(directories[j])
+    Filters eligiblepaths to rows where all hashes are in approved,
-            elif len(common) == len(parts_i) - 1 and len(parts_i) > 3:
+    then drops rows with fewer than min_hashes hashes.
-                group.append(directories[j])
+    """
-                used.add(directories[j])
+    approved_hashes = set(approved['sha256'])
-        used.add(path)
-        groups.append(group)
 
-    # Step 7: Map each original directory to its grouped prefix
+    def all_hashes_approved(row):
-    prefix_map = {dir: longest_common_prefix(group) for group in groups for dir in group}
+        return all(h in approved_hashes for h in row['sha256'])
-    df["grouped_directory"] = df["directory"].map(prefix_map)
 
-    # Step 8: Group the DataFrame by grouped_directory
+    filtered = eligiblepaths[eligiblepaths.apply(all_hashes_approved, axis=1)]
-    aggregation = {col: (lambda x: list(x)) for col in original_columns if col not in ["filename_x"]}
+    filtered = filtered[filtered['sha256'].apply(len) >= min_hashes]
-    aggregation.update({
+    
-        "directory": lambda x: list(x),
+    return filtered
-        "filename": lambda x: list(x)
-    })
 
-    grouped_df = df.groupby("grouped_directory", as_index=False).agg(aggregation)
+def inspect_parquet(path):
-
+    try:
-    # Step 9: Split into eligible and ineligible paths based on depth
+        df = pd.read_parquet(path)
-    grouped_df["depth"] = grouped_df["grouped_directory"].apply(lambda x: len(get_parts(x)))
+        print(f"✅ Successfully read: {path}")
-    path_eligible = grouped_df[grouped_df["depth"] > 2].drop(columns=["depth"])
+        print(f"📄 Columns: {df.columns.tolist()}")
-    path_ineligible = grouped_df[grouped_df["depth"] <= 2].drop(columns=["depth"])
+        print(f"🔢 Rows: {len(df)}")
-
-    # Step 10: Move entries from eligible to ineligible if grouped_directory contains excluded directories
-    mask = path_eligible["grouped_directory"].str.contains(r"(?i)(?:\\Users|\\c\$\\Users|inetpub\\wwwroot|windows\\temp)", na=False)
-    move_to_ineligible = path_eligible[mask]
-    path_eligible = path_eligible[~mask]
-    path_ineligible = pd.concat([path_ineligible, move_to_ineligible], ignore_index=True)
-
-    # Step 11: Deduplicate list elements in all columns
-    def deduplicate_lists(df):
-        for col in df.columns:
-            if df[col].apply(lambda x: isinstance(x, list)).all():
-                df[col] = df[col].apply(lambda x: list({str(item): item for item in chain.from_iterable(x if isinstance(x[0], list) else [x])}.values()))
         return df
+    except Exception as e:
+        print(f"❌ Error reading {path}: {e}")
+        return pd.DataFrame()
 
-    path_eligible = deduplicate_lists(path_eligible)
-    path_ineligible = deduplicate_lists(path_ineligible)
 
-    return path_eligible, path_ineligible
+def regulator(paths, case_insensitive=True):
+    """
+    Build a Python raw string regex that matches any of the given Windows path fragments.
+    """
+    escaped = [re.escape(p) for p in paths]
+    pattern = "(?:" + "|".join(escaped) + ")"
+    if case_insensitive:
+        pattern = pattern
+        print(f"Regulator is providing {pattern}")
+    return f'r"{pattern}"'

utils/policyfunctions.py

@@ -0,0 +1,64 @@
+#    Copyright (C) 2025 James Brotosky, Brandon Wickline
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import requests
+import json
+import os
+import utils.pretty as ct
+
+def addHash(policy, hash):
+    print(f"Adding the following hashes to {policy}:")
+    print(hash)
+
+def addPath(policy, hash):
+    print(f"Adding the following Path Exclusions to {policy}:")
+    print(hash)
+    
+def addHashReal(url, allowlistID, hashlist): 
+    endpoint = url + '/v1/hash/application/add'
+    print(ct.colorText("[+] Grabbing All Categories", "cyan"))
+    payload = {
+            "applicationid" : allowlistID,
+            "hashes" : hashlist
+    }
+    headers = {
+        "X-APIKey": os.getenv('APIKEY')
+    }
+    try:
+        response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+        response.raise_for_status()  # Raise an error for bad status codes
+        parse_text = json.loads(response.text)
+        print(parse_text)
+    except requests.exceptions.RequestException as e:
+        return {"error": str(e)}
+
+def addPathReal(url, grouplistID, pathlist): 
+    endpoint = url + '/v1/group/path/add'
+    print(ct.colorText("[+] Grabbing All Categories", "cyan"))
+    payload = {
+            "applicationid" : grouplistID,
+            "hashes" : pathlist
+        }
+    headers = {
+        "X-APIKey": os.getenv('APIKEY')
+    }
+    try:
+        response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+        response.raise_for_status()  # Raise an error for bad status codes
+        parse_text = json.loads(response.text)
+        print(parse_text)
+    except requests.exceptions.RequestException as e:
+        return {"error": str(e)}
+

utils/pretty.py

@@ -136,3 +136,78 @@ def style_dataframe_dark(df, output_html_path=None, overwrite=True):
         print(f"✅ Styled table saved to temporary file: {temp_path}")
     else:
         return styled_html
+
+
+def displayIntro():
+    
+    print(colorText(r"""
+                                                           ███                  
+                ████                                  ░████████                 
+               █████████████                      ███████████████               
+              █████████████████████          █████████████████████              
+              ███████████████████           ██████████████████████▓             
+             ███████████████████             ██████████████████████             
+            █████████████████████            ███████████████████████            
+            ████████████████████████████████████████████████████████            
+            █████████      ██              ██              █████████            
+            █████████      ██      ███      █              █████████            
+            █████████      ██      ████     █████      █████████████            
+            █████████      ██              ██████      █████████████            
+             ████████      ██             ███████      ████████████░            
+              ███████      ██      ██▓     ██████      ████████████             
+               ██████      ██      ████     █████      ███████████              
+                █████████████████████████████████████████████████               
+                  ▒████████████████████      ██████████████████                 
+                     ███████████████████      ███████████████▒                  
+                         ███████████████      █████████████                     
+                                ██████████    ███████████                       
+                                             ████████                           
+                                             ████                               
+""", "yellow"))
+    print(colorText(r"""
+   _____  .__       .__                 __     ___________           .__          
+  /  _  \ |__|______|  |   ____   ____ |  | __ \__    ___/___   ____ |  |   ______
+ /  /_\  \|  \_  __ \  |  /  _ \_/ ___\|  |/ /   |    | /  _ \ /  _ \|  |  /  ___/
+/    |    \  ||  | \/  |_(  <_> )  \___|    <    |    |(  <_> |  <_> )  |__\___ \ 
+\____|__  /__||__|  |____/\____/ \___  >__|_ \   |____| \____/ \____/|____/____  >
+        \/                           \/     \/                                 \/ 
+""", "cyan"))
+    print(colorText("=================================================================================", "cyan"))
+    print(colorText("======================== Welcome to the Airlock API Tool ========================", "cyan"))
+    print(colorText("=================================================================================", "cyan"))
+
+
+def areYouSure():
+    print(colorText(f"*******************************************************************************************************************************************","red"))
+    print(colorText(f"*=========================================================================================================================================*","yellow"))
+    print(colorText(f"*=========================================================================================================================================*","red"))
+    print(colorText(f"*-------------This program will now begin to make changes to the Airlock Console. Do you understand and agree to proceed? ----------------*", "yellow"))
+    print(colorText(f"*=========================================================================================================================================*","red"))
+    print(colorText(f"*=========================================================================================================================================*","yellow"))
+    print(colorText(f"*******************************************************************************************************************************************","red"))
+
+def locked():
+
+    print(colorText(r"""
+        ████████████████████████████████████████████████████████████████        
+      ███                                                              ██       
+      ██                             ██████                            ███      
+      ██                          ████████████                         ███      
+      ██                        ████        ███                        ███      
+      ██                        ███          ███                       ███      
+      ██                        ███          ███                       ███      
+      ██                     ▒████████████████████                     ███      
+      ██                     ██████████████████████                    ███      
+      ██                     ██████████████████████                    ███      
+      ██                     ██████████████████████                    ███      
+      ██                     ██████████████████████                    ███      
+      ██                     ██████████████████████                    ███      
+      ██                                                               ███      
+      ███                                                              ███      
+      ████████████████████████████████████████████████████████████████████            
+      ▒██████████████████████████████████████████████████████████████████▒  
+                                     ▒████                                      
+                                     ▒████                                      
+                  ▓██████████████████████████████████████████                   
+                 █████████████████████████████████████████████░  
+""", "yellow"))