Merge pull request 'Zar-Branch' (#5) from Zar-Branch into master

Reviewed-on: brotoskyj/AirlockTools#5

AirlockTools.py

@@ -2,7 +2,10 @@ import dotenv
 import os
 import utils.getdeviceevents
 import utils.allowlist
+import utils.hashfunctions
+import utils.pathfunctions
 import urllib3
+import pandas as pd
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -26,9 +29,64 @@ def menu():
 	while True:
 		choice = input("Enter Menu Item: ")
 		if choice == '1':
-			utils.getdeviceevents.devicehistory(url)
+			utils.getdeviceevents.devicehistory(url,False)
 		if choice == '2':
-			utils.allowlist.allowlistexechistories(url)
+			utils.allowlist.allowlistexechistories(url,False)
+		if choice == '3':
+			executionhist = utils.allowlist.allowlistexechistories(url,True)
+			print(executionhist)
+			aggregated = utils.hashfunctions.aggregateHashes(executionhist)
+			print(aggregated)
+			augmented = utils.hashfunctions.augmentAggregatedHashes(url,aggregated)
+			print(augmented)
+
+			augmented.to_html("augmentedlist.html", index=False)
+
+			badpublisherlist = []
+			categorized = utils.hashfunctions.categorizeHashes(augmented, 5, badpublisherlist)
+			categorized[0].to_html("needsreview.html", index=False)
+			categorized[1].to_html("approved.html", index=False)
+			categorized[2].to_html("remaining.html", index=False)
+
+		if choice == '4':
+			html_file = "augmentedlist.html"
+			augmented_df = pd.read_html(html_file)
+			print(augmented_df)
+			combined_df = pd.concat(augmented_df, ignore_index=True)
+
+			path_eligible, path_ineligible = utils.pathfunctions.filepathInitialGroup(combined_df)
+			path_eligible.to_html("EligblePaths.html", index=False)
+			path_ineligible.to_html("IneligiblePaths.html",index=False)
+		
+		if choice == '5':
+			
+			executionhist = utils.allowlist.allowlistexechistories(url,True)
+			print(executionhist)
+
+			aggregated = utils.hashfunctions.aggregateHashes(executionhist)
+			print(aggregated)
+
+			augmented = utils.hashfunctions.augmentAggregatedHashes(url,aggregated)
+			print(augmented)
+
+			augmented.to_html("augmentedlist.html", index=False)
+			html_file = "augmentedlist.html"
+			augmented_df = pd.read_html(html_file)
+			combined_df = pd.concat(augmented_df, ignore_index=True)
+
+			path_eligible, path_ineligible = utils.pathfunctions.filepathInitialGroup(combined_df)
+			path_eligible.to_html("EligblePaths.html", index=False)
+			path_ineligible.to_html("IneligiblePaths.html",index=False)
+		
+			badpublisherlist = []
+			categorized = utils.hashfunctions.categorizeHashes(augmented, 5, badpublisherlist)
+			categorized[0].to_html("needsreview.html", index=False)
+			categorized[1].to_html("approved.html", index=False)
+			categorized[2].to_html("remaining.html", index=False)
+			
+	
+			
+
 
 if __name__ == "__main__":
     apivalidation()

utils/allowlist.py

@@ -3,7 +3,7 @@ import requests
 import json
 import os
 
-def allowlistexechistories(url): 
+def allowlistexechistories(url, outputjson: bool): 
     endpoint = url + '/v1/group'
     print("[+] Grabbing All Policies")
     payload = {}
@@ -22,13 +22,17 @@ def allowlistexechistories(url):
     choice = int(choice) - 1
     endpoint = url + '/v1/logging/exechistories'
     payload_dict = {
-        "type":[1],
+        "type":[1, 2, 6, 7],
         "checkpoint":"68a153c23963989b484541b4",
         "policy": [policiesnames[choice]]
     }
     payload = json.dumps(payload_dict)
     print(payload)
     response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+    
+    if outputjson == True:
+        return response
+    
     parse_text = json.loads(response.text)
     for item in parse_text['response']['exechistories']:
         print(item['checkpoint'])

utils/getdeviceevents.py

@@ -3,7 +3,7 @@ import requests
 import json
 import os
 
-def devicehistory(url):
+def devicehistory(url, outputjson: bool):
 	endpoint = url + '/v1/getexechistory'
 	print("\n")
 	print("1. Today")
@@ -43,7 +43,12 @@ def devicehistory(url):
 	}
 
 	response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
+	
+	if outputjson == True:
+		return response
+
 	parse_text = json.loads(response.text)
+
 	for block in parse_text['response']['exechistory']:
 		print(f"Command: {block['commandline']}")
 		print(f"Date: {block['datetime']}")

utils/hashfunctions.py

@@ -1,21 +1,28 @@
 import pandas as pd
 import requests
 import os
+import json
 
-def aggregateHashes(executions_json: dict) -> pd.DataFrame:
+
+def aggregateHashes(executions_json) -> pd.DataFrame:
     """
     Takes the executions,  aggregates all the data with sha256 as primary, then returns aggregated dataframe
     """
+    data = executions_json.json()
+    df = pd.DataFrame(data["response"]["exechistories"])
 
-    exechistories = executions_json.get("response", {}).get("exechistories", [])
-    df = pd.DataFrame(exechistories)
-    
     if df.empty:
-        return df 
-    
-    # Aggregate by sha256 - keep all entries in lists
-    agg_df = df.groupby("sha256").agg(lambda x: list(x)).reset_index()
-    
+         return df
+
+    # Aggregate by sha256, deduplicate lists, and preserve order
+    agg_df = df.groupby("sha256").agg(lambda x: list(dict.fromkeys(x))).reset_index()
+
+    # Add a column for the number of unique hostnames
+    agg_df["num_devices"] = agg_df["hostname"].apply(len)
+
+    # Sort by num_devices in descending order
+    agg_df = agg_df.sort_values("num_devices", ascending=False)
+
     return agg_df
 
 def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
@@ -23,14 +30,18 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
     Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash,
     aggregate applications and baselines into lists, then merges results back into agg_df to create a 
     """
-    endpoint = url + 'v1/hash/query'
-    payload = agg_df['sha256'].tolist()
+    endpoint = url + '/v1/hash/query'
+    payload = {
+            "hashes": agg_df['sha256'].tolist()
+    }
+    
     headers = {"X-APIKey": os.getenv('APIKEY')}
-
-    response = requests.post(endpoint, headers=headers, json=payload, verify=False)
+    payload = json.dumps(payload)
+  
+    response = requests.post(endpoint, headers=headers, data=payload, verify=False)
     data = response.json()  
     results = data.get("response", {}).get("results", [])
-
+    
     rows = []
     for res in results:
         row = {"sha256": res.get("sha256"), "result": res.get("result")}
@@ -56,29 +67,27 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
     aug_df = agg_df.merge(df_api, on="sha256", how="left")
 
     return aug_df
+ 
+def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
+   if untrusted_publishers is None:
+       untrusted_publishers = []
 
-def categorize_hashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
-    """
-    Categorize hashes into needsreview, approved, and remaining based on publisher and threat level.
-    
-    """
-    if untrusted_publishers is None:
-        untrusted_publishers = []
+   df = aug_df.copy()
+   def reputationtool(row, threat_tolerance):
+        if row["reputation_scannermatch"] == "N/A":
+            return True
+        try:
+            if int(row["reputation_scannermatch"]) > threat_tolerance:
+                return True
+        except (ValueError, TypeError):
+            pass
+        return False
 
-    # Flatten threatlevel from nested reputation dict
-    df = df.copy()  
-    df["threatlevel"] = df["reputation"].apply(lambda x: x.get("threatlevel") if pd.notnull(x) else None)
+   mask_needsreview = (df["publisher_y"] == "Not Signed") & df.apply(lambda row: reputationtool(row, threat_tolerance), axis=1)
+   mask_approved = (df["publisher_y"] != "Not Signed") & (~df["publisher_y"].isin(untrusted_publishers))
 
-    # Masks for each category
-    mask_needsreview = (df["publisher"] == "Not Signed") & (df["threatlevel"] > threat_tolerance)
-    mask_approved = (df["publisher"] != "Not Signed") & (~df["publisher"].isin(untrusted_publishers))
+   needsreview_df = df[mask_needsreview]
+   approved_df = df[mask_approved]
+   remaining_df = df[~(mask_needsreview | mask_approved)]
 
-    # Create DataFrames for each category
-    needsreview_df = df[mask_needsreview].drop(columns=["threatlevel"])
-    approved_df = df[mask_approved].drop(columns=["threatlevel"])
-    remaining_df = df[~(mask_needsreview | mask_approved)].drop(columns=["threatlevel"])
-
-    return needsreview_df, approved_df, remaining_df
-
-def approve_hashes(approved_df: pd.DataFrame):
-    pass
+   return needsreview_df, approved_df, remaining_df

utils/pathfunctions.py

@@ -0,0 +1,100 @@
+import pandas as pd
+import os
+from itertools import chain
+
+def filepathInitialGroup(df: pd.DataFrame):
+    original_columns = df.columns.tolist()
+
+    # Step 1: Split comma-separated filepaths into lists
+    df["filename_x"] = df["filename_x"].str.split(",")
+
+    # Step 2: Explode the list so each filepath becomes its own row
+    df = df.explode("filename_x", ignore_index=True)
+
+    # Step 3: Clean up whitespace
+    df["filename_x"] = df["filename_x"].str.strip()
+
+    # Step 4: Extract directory and filename from each filepath
+    df["directory"] = df["filename_x"].apply(lambda x: os.path.dirname(x) if pd.notna(x) else "")
+    df["filename"] = df["filename_x"].apply(lambda x: os.path.basename(x) if pd.notna(x) else "")
+
+    # Step 5: Drop the original raw filepath column
+    df = df.drop(columns=["filename_x"])
+
+    # Helper functions for path manipulation
+    def get_parts(path):
+        return path.strip("\\").split("\\")
+
+    def join_parts(parts):
+        return "\\".join(parts)
+
+    def longest_common_prefix(paths):
+        split_paths = [get_parts(p) for p in paths]
+        min_len = min(len(p) for p in split_paths)
+        prefix = []
+        for i in range(min_len):
+            current = split_paths[0][i]
+            if all(p[i] == current for p in split_paths):
+                prefix.append(current)
+            else:
+                break
+        return join_parts(prefix)
+
+    # Step 6: Group directories by shared prefix using custom logic
+    directories = df["directory"].tolist()
+    groups = []
+    used = set()
+
+    for i, path in enumerate(directories):
+        if path in used:
+            continue
+        group = [path]
+        parts_i = get_parts(path)
+        for j in range(i + 1, len(directories)):
+            parts_j = get_parts(directories[j])
+            common = os.path.commonprefix([parts_i, parts_j])
+            if (len(parts_i) > 3 and len(common) >= 3) or (len(parts_i) == 3 and len(common) >= 2):
+                group.append(directories[j])
+                used.add(directories[j])
+            elif len(common) == len(parts_i) - 1 and len(parts_i) > 3:
+                group.append(directories[j])
+                used.add(directories[j])
+        used.add(path)
+        groups.append(group)
+
+    # Step 7: Map each original directory to its grouped prefix
+    prefix_map = {dir: longest_common_prefix(group) for group in groups for dir in group}
+    df["grouped_directory"] = df["directory"].map(prefix_map)
+
+    # Step 8: Group the DataFrame by grouped_directory
+    aggregation = {col: (lambda x: list(x)) for col in original_columns if col not in ["filename_x"]}
+    aggregation.update({
+        "directory": lambda x: list(x),
+        "filename": lambda x: list(x)
+    })
+
+    grouped_df = df.groupby("grouped_directory", as_index=False).agg(aggregation)
+
+    # Step 9: Split into eligible and ineligible paths based on depth
+    grouped_df["depth"] = grouped_df["grouped_directory"].apply(lambda x: len(get_parts(x)))
+    path_eligible = grouped_df[grouped_df["depth"] > 2].drop(columns=["depth"])
+    path_ineligible = grouped_df[grouped_df["depth"] <= 2].drop(columns=["depth"])
+
+    # Step 10: Move entries from eligible to ineligible if grouped_directory contains 'C:\Users' or 'c$\Users'
+    mask = path_eligible["grouped_directory"].str.contains(r"(?i)(?:\\Users|\\c\$\\Users)")
+    move_to_ineligible = path_eligible[mask]
+    path_eligible = path_eligible[~mask]
+    path_ineligible = pd.concat([path_ineligible, move_to_ineligible], ignore_index=True)
+
+    # Step 11: Deduplicate list elements in all columns
+    def deduplicate_lists(df):
+        for col in df.columns:
+            if df[col].apply(lambda x: isinstance(x, list)).all():
+                df[col] = df[col].apply(lambda x: list({str(item): item for item in chain.from_iterable(x if isinstance(x[0], list) else [x])}.values()))
+        return df
+
+
+    path_eligible = deduplicate_lists(path_eligible)
+    path_ineligible = deduplicate_lists(path_ineligible)
+
+    return path_eligible, path_ineligible