Merge branch 'Zar-Branch' of https://git.racooncity.org/brotoskyj/Airlocktools into Zar-Branch

AirlockTools.py

@@ -19,11 +19,11 @@ import utils.getdeviceevents
 import utils.allowlist
 import utils.hashfunctions
 import utils.pathfunctions
-import utils.allowfunctions
 import utils.colortext as ct
 import urllib3
 import pandas as pd
 
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 dotenv.load_dotenv()
 
@@ -121,7 +121,7 @@ def menu_prepare_to_enforce():
 	#If the directorys where we're going to store our output dont exist, make them. 
 	if not os.path.exists("dataframe_html"): os.makedirs("dataframe_html")
 	if not os.path.exists("dataframe_csv"): os.makedirs("dataframe_csv")
-	if not os.path.exists("manuallyapproved"): os.makedirs("approvals")
+	if not os.path.exists("approvals"): os.makedirs("approvals")
 
 	df_aggregated_combo = pd.DataFrame()
 	while True:
@@ -177,25 +177,14 @@ def menu_prepare_to_enforce():
 			print(ct.colorText("  [✓]  This step has been completed","green"))
 		else:
 			print(ct.colorText("  [✗]  This step has not been completed","red"))
-
+			
-		print(ct.colorText(f"7. Manually review the files \\dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv and dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", "cyan"))
-		print(ct.colorText("     Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.", "cyan"))
-		print(ct.colorText("     If metarules need to be created, please make note of them, and remove the row from the csv.", "cyan"))
-		print(ct.colorText("     When complete, move both csv files to the directory 'manuallyapproved' and choose this option to combine these approved hashes with the automatically approved hashes", "cyan"))
-
-		if os.path.isfile(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv"):
-			print(ct.colorText("  [✓]  This step has been completed","green"))
-		else:
-			print(ct.colorText("  [✗]  This step has not been completed","red"))
-
-		"""	
 		print(ct.colorText("7. Compare potential path exclusions with allowed hashes", "cyan"))
 				   
 		if os.path.exists(f"dataframe_csv\\df_allowed_paths_{first_policy}_{second_policy}.csv") == True:
 			print(ct.colorText("  [✓]  This step has been completed","green"))
 		else:
 			print(ct.colorText("  [✗]  This step has not been completed","red"))
-		"""
+		
 
 		print(ct.colorText("Q. Quit", "cyan"))
 
@@ -279,14 +268,6 @@ def menu_prepare_to_enforce():
 			else:
 				print(ct.colorText(f"Please Augment your data with hash threat info using step 4 prior to attempting this step","red"))
 
-		elif choice == "7":
-			if os.path.isfile(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv"):
-				df1 = tryToReadCSV(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv")
-				df2 = tryToReadCSV(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv")
-				df_all_approved_hashes = pd.concat([df1 , df2], ignore_index=True)
-				df_aggregated_combo.to_html(f"dataframe_html\\df_all_approved_hashes_{first_policy}_{second_policy}.html", index=False)
-				df_aggregated_combo.to_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
-		"""
 		elif choice == "7":
 			if os.path.exists(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.exists(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv") and os.path.exists(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv"):
 				allowpaths = utils.allowfunctions.filter_and_drop(pd.read_csv(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv"),tryToReadCSV(f"dataframe_csv\\df_path_eligible_{first_policy}_{second_policy}.csv"), path_exclusion_constant)
@@ -295,8 +276,7 @@ def menu_prepare_to_enforce():
 				print(ct.colorText(f"Allowable paths determined","green"))
 			else:
 				print(ct.colorText(f"Please complete step 6 prior to attempting this step","red"))
-		"""
+		
-
 		elif choice == "Q":
 			break
 		

hashtest.py

@@ -0,0 +1,5 @@
+hashes = ''
+while True:
+    inputhash = input("Hash: ")
+    hashes = hashes + ',' + inputhash
+    print(hashes)

utils/allowlist.py

@@ -17,7 +17,6 @@ import requests
 import json
 import os
 import time
-import utils.colortext as ct
 
 def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): 
    
@@ -73,4 +72,71 @@ def listPolicies(url):
         policyids.append(list['groupid'])
     choice = input(ct.colorText("Select Policy Group: ", "white"))
     choice = int(choice) - 1
-    return choice, policiesnames
+    checkpoint = '000000000000000000000000'
+    json_output = {'error': 'Success', 'response': {'exechistories': []}}
+    while True:
+        json_response_data = checkpoint_stomper(checkpoint, url, policiesnames[choice], headers)
+        if not json_response_data['response']['exechistories']:
+            break
+        for index, item in enumerate(json_response_data['response']['exechistories']):
+            if index == len(json_response_data['response']['exechistories']) -1:
+                checkpoint = item['checkpoint']
+                print(f"Date Greater than 30 Days, Stepping to new Checkpoint. {item['checkpoint']}")
+            else:
+                if (datetime.date.today() - datetime.timedelta(days=10) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
+                    pass
+                else:
+                    #json_output['response']['exechistories'].append(json_response_data['response']['exechistories'][1])
+                    for output in json_response_data['response']['exechistories']:
+                        json_output['response']['exechistories'].append(output)
+    json_output = json.dumps(json_output)
+    if outputjson == True:
+        return json_output
+    #endpoint = url + '/v1/logging/exechistories'
+    #payload_dict = {
+    #    "type":[1, 2, 6, 7],
+    #    "checkpoint":"000000000000000000000000",
+    #    "policy": [policiesnames[choice]]
+    #}
+    #payload = json.dumps(payload_dict)
+    #print(payload)
+    #response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+    #parse_text = json.loads(response.text)
+    #text_response = checkpoint_stomper(parse_text['response']['exechistories'], url, policiesnames[choice])
+#
+    #if outputjson == False:
+    #    return response
+    #
+    #parse_text = json.loads(response.text)
+    #
+    #for item in parse_text['response']['exechistories']:
+    #    print(item['checkpoint'])
+    #    print(item['datetime'])
+    #    print(item['hostname'])
+    #    print(item['filename'])
+    #    checkpoint_stomper(item['checkpoint'], endpoint, headers, policiesnames[choice])
+
+def checkpoint_stomper(checkpoint, url, policy, headers):
+    endpoint = url + '/v1/logging/exechistories'
+    payload_dict = {
+        "type":[1,2,6,7],
+        "checkpoint": checkpoint,
+        "policy": [policy]
+    }
+    payload = json.dumps(payload_dict)
+    response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+    parse_text = json.loads(response.text)
+    return parse_text
+    #for index, item in enumerate(parse_text):
+    #    if index == len(parse_text) - 1:
+    #        checkpoint = item['checkpoint']
+    #        print(f"Time: {item['datetime']} Checkpoint: {item['checkpoint']}")
+    #        response_fuzzer(checkpoint, url, policyname)
+    #    else:
+    #        if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace( ' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
+    #            pass
+    #        else:
+    #            response_fuzzer(checkpoint, url, policyname)
+
+
+    print("Finished")

utils/hashfunctions.py

@@ -90,38 +90,28 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
 
     df = aug_df.copy()
 
-    def reputationtool(row):
+    def reputationtool(row, threat_tolerance):
-        val = row["reputation_scannermatch"]
+        if row["reputation_scannermatch"] == "N/A":
-        if pd.isna(val) or val == "N/A":
+            return True
-            return row["publisher_y"] == "Not Signed"
         try:
             return int(val) > threat_tolerance
         except (ValueError, TypeError):
             return row["publisher_y"] == "Not Signed"
 
-    df["reputation_flag"] = df.apply(reputationtool, axis=1)
+    mask_needsreview = (df["publisher_y"] == "Not Signed") & df.apply(lambda row: reputationtool(row, threat_tolerance), axis=1)
-
-    mask_needsreview = (
-        ((df["publisher_y"] == "Not Signed") & df["reputation_flag"]) |
-        (df["reputation_status"] == "UNKNOWN")
-    )
 
     mask_approved = (
-        (
+    ((df["publisher_y"] != "Not Signed") & (~df["publisher_y"].isin(untrusted_publishers))) &
-            (df["publisher_y"] != "Not Signed") &
+    (df["publisher_y"] != "Not Signed")  # explicitly signed
-            ~df["publisher_y"].isin(untrusted_publishers) &
+    ) | (
-            ~df["reputation_status"].isna()
+    (df["publisher_y"] == "Not Signed") &
-        ) |
+    (~df.apply(lambda row: reputationtool(row, threat_tolerance), axis=1)) &
-        (
+    (~df["publisher_y"].isin(untrusted_publishers))  # exclude untrusted even if unsigned
-            (df["publisher_y"] == "Not Signed") &
-            ~df["reputation_flag"] &
-            ~df["publisher_y"].isin(untrusted_publishers) &
-            ~df["reputation_status"].isna()
-        )
     )
 
+
     needsreview_df = df[mask_needsreview]
     approved_df = df[mask_approved]
-    unapproved_df = df[~(mask_needsreview | mask_approved)]
+    remaining_df = df[~(mask_needsreview | mask_approved)]
 
-    return needsreview_df, approved_df, unapproved_df
+    return needsreview_df, approved_df, remaining_df

utils/pathfunctions.py

@@ -57,21 +57,45 @@ def filepathInitialGroup(df: pd.DataFrame):
                 break
         return join_parts(prefix)
 
-    # Step 6: Group directories by shared prefix
+    # Step 6: Group directories by shared prefix using custom logic
+    """
+    Loop through each directory path
+    directories: list of all directory paths.
+    groups: will hold lists of grouped directories.
+    used: tracks which directories have already been grouped.
+    """
     directories = df["directory"].tolist()
     groups = []
     used = set()
 
+    #For Each directory, compare it with others
+    """
+    Skip if already grouped.
+    Start a new group with the current path.
+    parts_i is the list of folder names in the path (e.g., ["C:", "Users", "John", "Documents"]).
+    """
+    
     for i, path in enumerate(directories):
         if path in used:
             continue
         group = [path]
         parts_i = get_parts(path)
 
+    #Compare with all other directories: For each other directory, split it into parts and find the common prefix (shared folder structure).
+        """
+        Logic:
+        If the directory is deep (>3 parts) and shares at least 3 parts → group it.
+        If it's exactly 3 parts long and shares at least 2 → group it.
+        Or, if it shares all but one part and is deep → group it.
+        These rules are designed to:
+        Group directories that are closely related in structure.
+        Avoid grouping unrelated paths that just happen to start similarly.
+        """  
+
         for j in range(i + 1, len(directories)):
             parts_j = get_parts(directories[j])
             common = os.path.commonprefix([parts_i, parts_j])
-
+        #Apply grouping rules
             if (len(parts_i) > 3 and len(common) >= 3) or (len(parts_i) == 3 and len(common) >= 2):
                 group.append(directories[j])
                 used.add(directories[j])
@@ -99,7 +123,7 @@ def filepathInitialGroup(df: pd.DataFrame):
     path_eligible = grouped_df[grouped_df["depth"] > 2].drop(columns=["depth"])
     path_ineligible = grouped_df[grouped_df["depth"] <= 2].drop(columns=["depth"])
 
-    # Step 10: Move entries from eligible to ineligible if grouped_directory contains excluded directories
+    # Step 10: Move entries from eligible to ineligible if grouped_directory contains 'C:\Users' or 'c$\Users'
     mask = path_eligible["grouped_directory"].str.contains(r"(?i)(?:\\Users|\\c\$\\Users|inetpub\\wwwroot|windows\\temp)", na=False)
     move_to_ineligible = path_eligible[mask]
     path_eligible = path_eligible[~mask]