mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

MemOpt #18

Merged
mysticmomba merged 30 commits from MemOpt into master 2025-09-02 17:53:43 -04:00
Conversation 0 Commits 30 Files Changed 9
+914 -321
9 changed files with 914 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -2,3 +2,5 @@
*.html
*.csv
*__pycache__*
*.parquet
chunkinator.json
AirlockTools.py
+454 -155
View File
@@ -14,49 +14,69 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import dotenv
import gc
import json
import os
import utils.getdeviceevents
import pandas as pd
import urllib3
import utils.allowlist
import utils.getdeviceevents
import utils.hashfunctions
import utils.pathfunctions
import utils.policyfunctions
import utils.pretty as ct
import urllib3
import pandas as pd
import ast
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
dotenv.load_dotenv()
url = "https://172.17.22.240:3129"
#Constants
url = os.getenv('url')
badpublisherlist = ["Brave Software, Inc.", "Zoom Video Communications, Inc."]
badpathparts = ["users", "inet\\wwwroot", "windows\\temp", "windows\\temp", "windows\\task", "windows\\system32", "startup", "windows\\fonts", "Recycle.Bin", "AppData", "programdata"]
path_exclusion_constant = 3
min_files_for_path = 4
threat_tolerance_constant = 4
def apivalidation():
print(ct.colorText(r"""
_____ .__ .__ __ ___________ .__
/ _ \ |__|______| | ____ ____ | | __ \__ ___/___ ____ | | ______
/ /_\ \| \_ __ \ | / _ \_/ ___\| |/ / | | / _ \ / _ \| | / ___/
/ | \ || | \/ |_( <_> ) \___| < | |( <_> | <_> ) |__\___ \
\____|__ /__||__| |____/\____/ \___ >__|_ \ |____| \____/ \____/|____/____ >
\/ \/ \/ \/
""", "cyan"))
print(ct.colorText("=================================================================================", "cyan"))
print(ct.colorText("======================== Welcome to the Airlock API Tool ========================", "cyan"))
print(ct.colorText("=================================================================================", "cyan"))
match os.getenv('APIKEY'):
case '':
print(ct.colorText("Please add your API Key to the .env file", "red"))
case _:
menu_main()
def tryToReadCSV(csv):
try:
df =pd.read_csv(csv)
if df.empty:
print(ct.colorText("Error: CSV file has headers but no data rows.", "red"))
else:
print(ct.colorText(f"Data loaded successfully from {csv}", "green"))
except pd.errors.EmptyDataError:
print(ct.colorText("Notice : CSV file is completely empty (no headers, no data), falling back to empty frame", "white"))
df = pd.DataFrame() # Create an empty DataFrame as fallback
return df
def tryToReadParquet(parquet):
try:
df = pd.read_parquet(parquet)
if df.empty:
print(ct.colorText("Error: Parquet file has headers but no data rows.", "red"))
else:
print(ct.colorText(f"Data loaded successfully from {parquet}", "green"))
except pd.errors.EmptyDataError:
print(ct.colorText("Notice : Parquet file is completely empty (no headers, no data), falling back to empty frame", "white"))
df = pd.DataFrame() # Create an empty DataFrame as fallback
return df
def deduplicate_list(lst):
seen = set()
return [x for x in lst if not (x in seen or seen.add(x))]
def menu_main():
while True:
print(ct.colorText("\n-----------------------------------", "magenta"))
print(ct.colorText("------------ Main Menu ------------", "magenta"))
print(ct.colorText("-----------------------------------", "magenta"))
ct.displayIntro();
print(ct.colorText("1. Get All Events for Single Device", "yellow"))
print(ct.colorText("2. Placeholder for Local Approval", "yellow"))
print(ct.colorText("3. Placeholder for Another Tool", "yellow"))
@@ -117,20 +137,25 @@ def menu_prepare_to_enforce():
first_policy = " "
second_policy = " "
allowlist_parent_name = " "
allowlist_child_name = " "
destination_name = " "
df_aggregated_combo = pd.DataFrame()
#If the directorys where we're going to store our output dont exist, make them.
if not os.path.exists("dataframe_html"): os.makedirs("dataframe_html")
if not os.path.exists("dataframe_csv"): os.makedirs("dataframe_csv")
if not os.path.exists("manuallyapproved"): os.makedirs("manuallyapproved")
if not os.path.exists("parquet"): os.makedirs("parquet")
if not os.path.exists("needs_approved"): os.makedirs("needs_approved")
if not os.path.exists("approved"): os.makedirs("approved")
if not os.path.exists("preflight"): os.makedirs("preflight")
df_aggregated_combo = pd.DataFrame()
while True:
print(ct.colorText("\n --------------------------------------------------------------------", "cyan"))
print(ct.colorText(" -------------------- Prepare to Enforce Policy ---------------------", "cyan"))
print(ct.colorText(" --------------------------------------------------------------------", "cyan"))
print(ct.colorText("\nSequentually follow these steps to prepare a policy for enforcement:", "white"))
print(ct.colorText("\n1. Choose which policy or policies to work with - : ", "cyan"))
print(ct.colorText("\n1. Choose which originating policy or policies to move to enforcement", "cyan"))
if first_policy == " " and second_policy == " ":
print(ct.colorText(f" [✗] No policies have been chosen","red"))
@@ -140,178 +165,467 @@ def menu_prepare_to_enforce():
print(ct.colorText(f" [✓] {first_policy} has been selected as Policy 1","green"))
print(ct.colorText(f" [✓] {second_policy} has been selected as Policy 2","green"))
print(ct.colorText("2. Pull and stage event history", "cyan"))
if os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") == True:
print(ct.colorText(f" [✓] This has been completed for {first_policy}","green"))
elif os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") == False:
print(ct.colorText(f" [✗] This step has not been completed","red"))
elif second_policy is not first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv") == True:
print(ct.colorText(f" [✓] This has been completed for {second_policy}","green"))
elif second_policy is not first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv") == False:
print(ct.colorText(f" [✓] This has not been completed for {second_policy}","red"))
print(ct.colorText("2. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", "cyan"))
print(ct.colorText("3. Combine Staged policies", "cyan"))
if os.path.exists(f"parquet\\execution_history_{first_policy}.parquet"):
print(ct.colorText(f" [✓] Execution history has been compiled for {first_policy}","green"))
elif not os.path.exists(f"parquet\\execution_history_{first_policy}.parquet"):
print(ct.colorText(f" [✗] Execution history has not been compiled for {first_policy}","red"))
elif second_policy is not first_policy and os.path.exists(f"parquet\\execution_history_{second_policy}.parquet"):
print(ct.colorText(f" [✓] Execution history has been compiled for {second_policy}","green"))
elif second_policy is not first_policy and not os.path.exists(f"parquet\\execution_history_{second_policy}.parquet"):
print(ct.colorText(f" [✗] Execution history has not been compiled for {second_policy}","red"))
if os.path.exists(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv") == True:
print(ct.colorText(" [✓] This step has been completed","green"))
if os.path.exists(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"):
print(ct.colorText(f" [✓] Hash Info has been added to the combined execution history", "green"))
else:
print(ct.colorText(" [✗] This step has not been completed","red"))
print(ct.colorText(f" [✗] Hash Info has not been added to the combined execution history", "red"))
print(ct.colorText("4. Add hash threat information to list of executions", "cyan"))
if os.path.exists(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv") == True:
print(ct.colorText(" [✓] This step has been completed","green"))
if os.path.exists(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet") and os.path.exists(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet") and os.path.exists(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet"):
print(ct.colorText(f" [✓] Hashes have been cateogrized", "green"))
else:
print(ct.colorText(" [✗] This step has not been completed","red"))
print(ct.colorText(f" [✗] Hashes have not been cateogrized", "red"))
print(ct.colorText("5. Categorize your hashes ", "cyan"))
if os.path.isfile(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv"):
print(ct.colorText(" [✓] This step has been completed","green"))
if os.path.exists(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet"):
print(ct.colorText(f" [✓] Execution history has been_combined_for {first_policy} and_{second_policy}", "green"))
else:
print(ct.colorText(" [✗] This step has not been completed","red"))
print(ct.colorText(f" [✗] Execution history has not been_combined_for {first_policy} and_{second_policy}", "red"))
print(ct.colorText(f"6. Manually review the files \\dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv and dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", "cyan"))
print(ct.colorText(f"3. Manually review the files:","cyan"))
print(ct.colorText(" 'needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv' and 'needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv'", "cyan"))
print(ct.colorText(" Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.", "cyan"))
print(ct.colorText(" If metarules need to be created, please make note of them, and remove the row from the csv.", "cyan"))
print(ct.colorText(" When complete, save both csv files to the directory 'manuallyapproved' and choose this option to combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan"))
print(ct.colorText(" When complete, save both csv files to the directory 'approved' and choose this option.","cyan"))
print(ct.colorText(" This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan"))
if os.path.isfile(f"dataframe_csv\\df_paths_needing_review_{first_policy}_{second_policy}.csv"):
print(ct.colorText(" [✓] This step has been completed","green"))
if os.path.exists(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv") and os.path.exists(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv"):
print(ct.colorText(" [✓] Reviewed hashes have been loaded","green"))
else:
print(ct.colorText(" [✗] This step has not been completed","red"))
print(ct.colorText(" [✗] Reviewed hashes have not been loaded","red"))
print(ct.colorText(f"7. Manually review the file df_paths_needing_review_{first_policy}_{second_policy}.csv", "cyan"))
if os.path.exists(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet"):
print(ct.colorText(" [✓] The combined approved hashes list has been generated","green"))
else:
print(ct.colorText(" [✗] The combined approved hashes list has not been generated","red"))
if os.path.exists(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet"):
print(ct.colorText(" [✓] Longest common filepaths have been generated and appended to hash info","green"))
else:
print(ct.colorText(" [✗] Longest common filepaths have not been generated","red"))
if os.path.exists(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
print(ct.colorText(" [✓] Path review list created","green"))
else:
print(ct.colorText(" [✗] Path review list has not been created","red"))
print(ct.colorText(f"4. Manually review the file 'needs_approved\\paths_needing_review_{first_policy}_{second_policy}.csv'", "cyan"))
print(ct.colorText(" Remove the rows containing path exclusions you do not approve of" , "cyan"))
print(ct.colorText(" When complete, save the csv file to the directory 'manuallyapproved' and choose this option to generate the proposed list of changes", "cyan"))
print(ct.colorText(" When complete, save the csv file to the directory 'approved'", "cyan"))
print(ct.colorText(" Preflight Lists will be generated", "cyan"))
if os.path.isfile(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv") and os.path.isfile(f"dataframe_csv\\df_hashdestination_{first_policy}_{second_policy}.csv"):
print(ct.colorText(" [✓] This step has been completed","green"))
if os.path.exists(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
print(ct.colorText(" [✓] Reviewed path list detected","green"))
else:
print(ct.colorText(" [✗] This step has not been completed","red"))
print(ct.colorText(" [✗] Path review list has not been detected","red"))
if os.path.exists(f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html"):
print(ct.colorText(" [✓] Preflight Path Exclusion List has been generated","green"))
else:
print(ct.colorText(" [✗] Preflight Path Exclusion List has not been generated","red"))
if os.path.exists(f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html"):
print(ct.colorText(" [✓] Preflight hash approval list has been generated","green"))
else:
print(ct.colorText(" [✗] Preflight hash approval list has not been generated","red"))
print(ct.colorText(f"5. Choose the destination policy and parent and child allow list", "cyan"))
if allowlist_child_name == " " and allowlist_parent_name== " ":
print(ct.colorText(f" [✗] No allowlists have been chosen","red"))
elif allowlist_parent_name != " " and allowlist_child_name != " " and allowlist_parent_name is allowlist_child_name:
print(ct.colorText(f" [✓] [✗] Only {allowlist_parent_name} has been selected this is unusual, but potentially valid case, double check before proceeding,", "yellow"))
elif allowlist_parent_name != " " and allowlist_child_name != " " and allowlist_parent_name is not allowlist_child_name:
print(ct.colorText(f" [✓] {allowlist_parent_name} has been selected as Parent Policy","green"))
print(ct.colorText(f" [✓] {allowlist_child_name} has been selected as Child Policy","green"))
if destination_name == " ":
print(ct.colorText(f" [✗] No destination policy has been chosen","red"))
else:
print(ct.colorText(f" [✓] destination policy is {destination_name}","green"))
print(ct.colorText(f"6. Liftoff ------------------------------------------------------", "cyan"))
print(ct.colorText(f" Apply path exclusions according to allowed and approved paths", "cyan"))
print(ct.colorText(f" Apply signed or attested hashes to Parent Allow List", "cyan"))
print(ct.colorText(f" Apply approved, but unsigned hashes to the Child Allow List", "cyan"))
print(ct.colorText("Q. Quit", "cyan"))
choice = input(ct.colorText("\nEnter your choice: ", "white"))
if choice == "1":
first_policy_tuple = utils.allowlist.listPolicies(url)
first_policy = first_policy_tuple[1][first_policy_tuple[0]]
choice, policynames, policyid = utils.allowlist.listPolicies(url)
first_policy = policynames[choice]
while True:
answer = input(ct.colorText(f"{"Do you want to load a second policy?"} (yes/no): ", "white").strip().lower())
if answer in ("yes", "y"):
second_policy_tuple = utils.allowlist.listPolicies(url)
second_policy = second_policy_tuple[1][second_policy_tuple[0]]
choice, policynames, policyid = utils.allowlist.listPolicies(url)
second_policy = policynames[choice]
break
elif answer in ("no", "n"):
second_policy_tuple = first_policy_tuple
second_policy = first_policy
break
else:
print(ct.colorText("Please answer with 'yes' or 'no'.", "red"))
elif choice == "2":
if not os.path.exists("dataframe_csv\\df_aggregated_{first_policy}.csv"):
executionhist_policy1 = utils.allowlist.pullPolicyExechistories(url,first_policy_tuple[0], first_policy_tuple[1],True)
df_aggregated_policy1 = utils.hashfunctions.aggregateHashes(executionhist_policy1)
df_aggregated_policy1.to_csv(f"dataframe_csv\\df_aggregated_{first_policy}.csv", index=False)
ct.style_dataframe_dark(df_aggregated_policy1, f"dataframe_html\\df_aggregated_{first_policy}.html")
print(ct.colorText(f"Staging of Exection history for policy: {first_policy} is complete","green"))
if not os.path.exists("dataframe_csv\\df_aggregated_{second_policy}.csv"):
executionhist_policy2 = utils.allowlist.pullPolicyExechistories(url,second_policy_tuple[0], second_policy_tuple[1],True)
df_aggregated_policy2 = utils.hashfunctions.aggregateHashes(executionhist_policy2)
df_aggregated_policy2.to_csv(f"dataframe_csv\\df_aggregated_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_aggregated_policy2, f"dataframe_html\\df_aggregated_{second_policy}.html")
print(ct.colorText(f"Staging of Exection history for policy: {second_policy} is complete","green"))
if not os.path.exists(f"parquet\\execution_history_{first_policy}.parquet"):
print(choice)
print(first_policy)
exe1 = utils.allowlist.pullPolicyExechistories(url, first_policy, 60, True)
data = json.loads(exe1)
executionhist_policy1 = pd.DataFrame(data["response"]["exechistories"])
if not executionhist_policy1.empty:
executionhist_policy1 = executionhist_policy1[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
executionhist_policy1 = executionhist_policy1.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
executionhist_policy1 = executionhist_policy1.sort_values(by=['sha256', 'filename'])
executionhist_policy1.to_parquet(f"parquet\\execution_history_{first_policy}.parquet", index=False)
print(ct.colorText(f"Staging of Execution history for policy: {first_policy} is complete", "green"))
del data
del exe1
del executionhist_policy1
gc.collect()
if not os.path.exists(f"parquet\\execution_history_{second_policy}.parquet"):
exe2 = utils.allowlist.pullPolicyExechistories(url,second_policy, 60, True)
data2 = json.loads(exe2)
executionhist_policy2 = pd.DataFrame(data2["response"]["exechistories"])
if not executionhist_policy2.empty:
executionhist_policy2 = executionhist_policy2[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
executionhist_policy2 = executionhist_policy2.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
executionhist_policy2 = executionhist_policy2.sort_values(by=['sha256', 'filename'])
executionhist_policy2.to_parquet(f"parquet\\execution_history_{second_policy}.parquet", index=False)
print(ct.colorText(f"Staging of Execution history for policy: {second_policy} is complete", "green"))
del executionhist_policy2
del data2
del exe2
gc.collect()
if not os.path.exists(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"):
combined_hashes = pd.DataFrame(columns=['sha256', 'publisher'])
hashes = []
try:
hash1 = pd.read_parquet(f"parquet\\execution_history_{first_policy}.parquet", columns=['sha256', 'publisher'])
utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{first_policy}.parquet")
if not hash1.empty:
hashes.append(hash1)
else:
print("⚠️ First dataframe is empty.")
except Exception as e:
print(f"❌ Error reading first Parquet file: {e}")
try:
hash2 = pd.read_parquet(f"parquet\\execution_history_{second_policy}.parquet", columns=['sha256', 'publisher'])
utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{second_policy}.parquet")
if not hash2.empty:
hashes.append(hash2)
else:
print("⚠️ Second dataframe is empty.")
except Exception as e:
print(f"❌ Error reading second Parquet file: {e}")
if hashes:
combined_hashes = pd.concat(hashes, ignore_index=True)
print(f"✅ Combined {len(combined_hashes)} hashes.")
else:
print("⚠️ No valid dataframes to combine.")
combined_hashes = combined_hashes.drop_duplicates(subset=['sha256'])
augmented_combo = utils.hashfunctions.augmentAggregatedHashes(url, combined_hashes)
numeric_reputation_cols = [
'reputation_scannermatch',
'reputation_scannercount',
'reputation_threatlevel'
]
for col in numeric_reputation_cols:
if col in augmented_combo.columns:
augmented_combo[col] = pd.to_numeric(augmented_combo[col].replace('N/A', pd.NA), errors='coerce')
augmented_combo = augmented_combo.rename(columns={'publisher_x': 'publisher'})
augmented_combo = augmented_combo[['sha256', 'publisher', 'description', 'productname', 'productversion',
'reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount',
'reputation_status', 'reputation_threatlevel', 'reputation_threatname',
'reputation_timestamp']]
augmented_combo = augmented_combo.sort_values(by=['publisher', 'description', 'productname'])
augmented_combo.to_parquet(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet", index=False)
del combined_hashes
del augmented_combo
gc.collect()
print(ct.colorText("Hash reputation info added to dataframe", "green"))
if not os.path.exists(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet"):
# Categorize the hashes
categorized = utils.hashfunctions.categorizeHashes(
pd.read_parquet(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"),
threat_tolerance_constant,
badpublisherlist
)
categorized[0].to_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet", index=False)
categorized[1].to_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet", index=False)
categorized[2].to_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet", index=False)
del categorized
gc.collect()
if not os.path.exists(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet"):
# Condense execution history
try:
exe1 = pd.read_parquet(f"parquet\\execution_history_{first_policy}.parquet")
utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{first_policy}.parquet")
if not exe1.empty:
condensed_exe1 = exe1.groupby('sha256').agg(lambda x: list(set(x))).reset_index()
else:
print("⚠️ First dataframe is empty.")
except Exception as e:
print(f"❌ Error reading first Parquet file: {e}")
try:
exe2 = pd.read_parquet(f"parquet\\execution_history_{second_policy}.parquet")
utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{second_policy}.parquet")
if not exe2.empty:
condensed_exe2 = exe2.groupby('sha256').agg(lambda x: list(set(x))).reset_index()
else:
print("⚠️ Second dataframe is empty.")
except Exception as e:
print(f"❌ Error reading second Parquet file: {e}")
if not exe1.empty and not exe2.empty:
condensed_combo = pd.concat([condensed_exe1, condensed_exe2], ignore_index=True)
print(f"✅ Combined {len(condensed_combo)} hashes.")
elif exe1.empty:
condensed_combo = condensed_exe2
elif exe2.empty:
condensed_combo = condensed_exe1
else:
print("⚠️ No valid dataframes to combine.")
condensed_combo.to_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet", index=False)
del condensed_combo
gc.collect()
if not os.path.exists(f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv"):
condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
needsapproval= pd.read_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet")
#Pull hash info for the entries in the needs approval table
needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
#Deduplicate lists in the columns
for col in needsapproval.columns:
if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
needsapproval[col] = needsapproval[col].apply(deduplicate_list)
#Rename Publisher, Keep and reorder columns we want
needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
needsapproval.to_csv(f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv",index=False)
needsapproval.to_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet",index=False)
ct.style_dataframe_dark(needsapproval, f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.html")
del needsapproval
del condensed_combo
gc.collect()
if not os.path.exists(f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv"):
condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
needsapproval= pd.read_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet")
#Pull hash info for the entries in the needs approval table
needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
#Deduplicate lists in the columns
for col in needsapproval.columns:
if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
needsapproval[col] = needsapproval[col].apply(deduplicate_list)
#Rename Publisher, Keep and reorder columns we want
needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
needsapproval.to_csv(f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv",index=False)
needsapproval.to_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet", index=False)
ct.style_dataframe_dark(needsapproval, f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.html")
del needsapproval
del condensed_combo
gc.collect()
if not os.path.exists(f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.html"):
condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
needsapproval= pd.read_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet")
#Pull hash info for the entries in the needs approval table
needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
#Deduplicate lists in the columns
for col in needsapproval.columns:
if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
needsapproval[col] = needsapproval[col].apply(deduplicate_list)
#Rename Publisher, Keep and reorder columns we want
needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
needsapproval.to_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet", index=False)
ct.style_dataframe_dark(needsapproval, f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.html")
del needsapproval
del condensed_combo
gc.collect()
elif choice == "3":
if second_policy is first_policy and os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv"):
df1 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{first_policy}.csv")
df_aggregated_combo = df1
df_aggregated_combo.to_csv(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\df_aggregated_combo_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Dataframes have been aggregated (combined)","green"))
elif os.path.exists(f"dataframe_csv\\df_aggregated_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\df_aggregated_{second_policy}.csv"):
df1 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{first_policy}.csv")
df2 = tryToReadCSV(f"dataframe_csv\\df_aggregated_{second_policy}.csv")
df_aggregated_combo = pd.concat([df1 , df2], ignore_index=True)
df_aggregated_combo.to_csv(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\df_aggregated_combo_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Dataframes have been aggregated (combined)","green"))
if os.path.exists(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv") and os.path.exists(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv"):
else:
print(ct.colorText(f"Please stage your data before attempting this step","red"))
if not os.path.exists(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet"):
elif choice == "4":
if os.path.exists(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv"):
df_augmented = utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\df_aggregated_combo_{first_policy}_{second_policy}.csv"))
df_augmented.to_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_augmented, f"dataframe_html\\df_augmented_combo_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Hash reputation info added to dataframe","green"))
df1 = tryToReadCSV(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv")
df2 = tryToReadCSV(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv")
else:
print(ct.colorText(f"Please combine your data with step 3 prior to attempting this step","red"))
all_approved_hashes = pd.concat([df1 , df2], ignore_index=True).sort_values(by=['sha256','filename'])
print(ct.colorText(f"Approved hash lists have been combined","green"))
elif choice == "5":
if os.path.exists(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv"):
categorized = utils.hashfunctions.categorizeHashes(pd.read_csv(f"dataframe_csv\\df_augmented_combo_{first_policy}_{second_policy}.csv"), threat_tolerance_constant, badpublisherlist)
all_approved_hashes.to_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet", index=False)
del all_approved_hashes
gc.collect()
categorized[0].to_csv(f"dataframe_csv\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(categorized[0], f"dataframe_html\\df_hashes_needing_approval_{first_policy}_{second_policy}.html")
if not os.path.exists(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet"):
all_approved_hashes = pd.read_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet")
print(ct.colorText(f"Beginning calculating longest common filepaths for path exceptions","green"))
grouped_df_view, df_with_groups_appended = utils.pathfunctions.export_groups_for_review(all_approved_hashes,"filename","longestcfp",min_files_for_path,path_exclusion_constant)
categorized[1].to_csv(f"dataframe_csv\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(categorized[1], f"dataframe_html\\df_automatically_approved_hashes_{first_policy}_{second_policy}.html")
df_with_groups_appended.to_parquet(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet", index=False)
categorized[2].to_csv(f"dataframe_csv\\df_unapproved_hashes__{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(categorized[2], f"dataframe_html\\df_unapproved_hashes_{first_policy}_{second_policy}.html")
forbidden = utils.pathfunctions.regulator(badpathparts, True)
forbidden_lcfp = grouped_df_view["longestcfp"].str.contains(forbidden, na=False)
grouped_df_view = grouped_df_view[~forbidden_lcfp]
print(ct.colorText(f"Removing forbidden filepaths for path exceptions","green"))
print(ct.colorText(f"Hashes have been categorized","green"))
grouped_df_view.to_parquet(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet", index=False)
grouped_df_view.to_csv(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(grouped_df_view, f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.html")
else:
print(ct.colorText(f"Please Augment your data with hash threat info using step 4 prior to attempting this step","red"))
elif choice == "6":
if os.path.exists(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv") and os.path.exists(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv"):
df1 = tryToReadCSV(f"manuallyapproved\\df_hashes_needing_approval_{first_policy}_{second_policy}.csv")
df2 = tryToReadCSV(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv")
df_all_approved_hashes = pd.concat([df1 , df2], ignore_index=True)
df_all_approved_hashes.to_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_all_approved_hashes, f"dataframe_html\\df_all_approved_hashes_{first_policy}_{second_policy}.html")
df_paths_needing_review, df_path_ineligible = utils.pathfunctions.filepathInitialGroup(pd.read_csv(f"dataframe_csv\\df_all_approved_hashes_{first_policy}_{second_policy}.csv"))
df_paths_needing_review.to_csv(f"dataframe_csv\\df_paths_needing_review_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_paths_needing_review, f"dataframe_html\\df_paths_needing_review_{first_policy}_{second_policy}.html")
df_path_ineligible.to_csv(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_path_ineligible, f"dataframe_html\\df_path_ineligible_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Eligible paths determined","green"))
del grouped_df_view
del df_with_groups_appended
gc.collect()
else:
print(ct.colorText(f"Please manually approve hashes prior to this step","red"))
elif choice == "7":
if os.path.exists(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv"):
df1 = tryToReadCSV(f"manuallyapproved\\df_paths_needing_review_{first_policy}_{second_policy}.csv")
df2 = tryToReadCSV(f"dataframe_csv\\df_path_ineligible_{first_policy}_{second_policy}.csv")
df3 = tryToReadCSV(f"manuallyapproved\\df_automatically_approved_hashes_{first_policy}_{second_policy}.csv")
df_hashdestination = utils.hashfunctions.destinationbuilder(df2,df3)
df_hashdestination.to_csv("dataframe_csv\\df_hashdestination_{first_policy}_{second_policy}.csv")
ct.style_dataframe_dark(df_hashdestination,f"dataframe_html\\df_hashdestination_{first_policy}_{second_policy}.html")
elif choice == "4":
else:
print(ct.colorText(f"Please manually approve suggested paths prior to this step","red"))
if os.path.exists(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet") and os.path.exists(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
if not os.path.exists(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet"):
df1 = pd.read_parquet(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet")
allowbyhash = utils.pathfunctions.mask_from_csv(df1, f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv","longestcfp")
pathexclusions = tryToReadCSV(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv")
pathexclusions.to_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet", index=False)
allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False)
easyview = allowbyhash.groupby('sha256').agg(list).reset_index()
ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html")
del df1
del allowbyhash
del pathexclusions
del easyview
gc.collect()
elif choice == "5":
print(ct.colorText(f"Please choose destination_name Policy for Path Exclusions","white"))
choice, policynames, policyid = utils.allowlist.listPolicies(url)
#print(allowlist_parent_tuple)
destination_name = policynames[choice]
destination_id = policyid[choice]
print(ct.colorText(f"Please choose Parent Allowlist for Known Hashes","white"))
choice, allowlists,allowid = utils.allowlist.listAllowlists(url)
#print(allowlist_parent_tuple)
allowlist_parent_name = allowlists[choice]
allowlist_parent_id = allowid[choice]
print(ct.colorText(f"Please choose Child Allowlist for Less-Known Hashes","white"))
choice, allowlists, allowid = utils.allowlist.listAllowlists(url)
#print(allowlist_child_tuple)
allowlist_child_name = allowlists[choice]
allowlist_child_id = allowid[choice]
elif choice == "6":
if os.path.exists(f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html") and os.path.exists(f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html") and allowlist_parent_name != " " and allowlist_child_name != " " and destination_name != " ":
pathexclusions = pd.read_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet")
allowbyhash = pd.read_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet")
ct.areYouSure()
confirmation = input(ct.colorText("Type 'I AGREE' to continue: ","white"))
if confirmation.strip().upper() == "I AGREE":
print(ct.colorText("Proceeding with the code...", "yellow"))
print(ct.colorText(f"Adding path exclusions to {destination_name}", "yellow"))
pathexcludelist = pathexclusions['longestcfp'].unique().tolist()
utils.policyfunctions.addPath(destination_id,pathexcludelist)
print(ct.colorText(f"Adding hashes to {allowlist_parent_name}", "yellow"))
allowlist_parenthashlist = allowbyhash[allowbyhash['reputation_status'] == 'KNOWN']['sha256'].unique().tolist()
utils.policyfunctions.addHash(allowlist_parent_id,allowlist_parenthashlist)
print(ct.colorText(f"Adding hashes to {allowlist_child_name}", "yellow"))
allowlist_childhashlist = allowbyhash[allowbyhash['reputation_status'] == 'UNKNOWN']['sha256'].unique().tolist()
utils.policyfunctions.addHash(allowlist_child_id, allowlist_childhashlist)
ct.locked()
exit()
else:
print(ct.colorText("Operation aborted. You MUST EXPLICITLY AGREE to proceed.", "red"))
break
elif choice == "Q":
break
@@ -319,21 +633,6 @@ def menu_prepare_to_enforce():
print(ct.colorText("Invalid choice. Please try again.", "red"))
def tryToReadCSV(csv):
try:
df =pd.read_csv(csv)
if df.empty:
print(ct.colorText("Error: CSV file has headers but no data rows.", "red"))
else:
print(ct.colorText("Data loaded successfully.", "green"))
except pd.errors.EmptyDataError:
print(ct.colorText("Notice : CSV file is completely empty (no headers, no data), falling back to empty frame", "white"))
df = pd.DataFrame() # Create an empty DataFrame as fallback
return df
if __name__ == "__main__":
apivalidation()
allowlistandgroup.py
+22
View File
@@ -0,0 +1,22 @@
import requests
import dotenv
import json
import os
import utils.pretty as ct
url = 'https://172.17.22.240:3129'
policiesnames = []
policyids=[]
dotenv.load_dotenv()
endpoint = url + '/v1/application'
print(ct.colorText("[+] Grabbing All Allowlists", "cyan"))
payload = {}
headers = {
"X-APIKey": os.getenv('APIKEY')
}
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
parse_text = json.loads(response.text)
for index, list in enumerate(parse_text['response']['applications'], start=1):
if index >= 38:
print(list)
#Need else and catch for upper bound
utils/allowfunctions.py
-16
View File
@@ -1,16 +0,0 @@
import pandas as pd
def filter_and_drop(approved, eligiblepaths, min_hashes):
"""
Filters eligiblepaths to rows where all hashes are in approved,
then drops rows with fewer than min_hashes hashes.
"""
approved_hashes = set(approved['sha256'])
def all_hashes_approved(row):
return all(h in approved_hashes for h in row['sha256'])
filtered = eligiblepaths[eligiblepaths.apply(all_hashes_approved, axis=1)]
filtered = filtered[filtered['sha256'].apply(len) >= min_hashes]
return filtered
utils/allowlist.py
+84 -29
View File
@@ -16,48 +16,67 @@ import datetime
import requests
import json
import os
import time
import utils.pretty as ct
import ijson
import os
from bson import ObjectId
import datetime
def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool):
headers = {
"X-APIKey": os.getenv('APIKEY')
}
checkpoint = '000000000000000000000000'
def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
file_path = 'chunkinator.json'
if not os.path.exists(file_path):
with open(file_path, 'w') as file:
json.dump({'error': 'Success', 'response': {'exechistories': []}}, file)
print(f"File '{file_path}' has been crated.")
else:
print(f"File '{file_path}' already exists.")
headers = {"X-APIKey": os.getenv('APIKEY')}
checkpoint = str(skipback(days))
json_output = {'error': 'Success', 'response': {'exechistories': []}}
while True:
json_response_data = checkpoint_stomper(checkpoint, url, policiesnames[choice], headers)
if not json_response_data['response']['exechistories']:
json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers)
histories = json_response_data['response']['exechistories']
if not histories:
break
array_dividend = max(round(len(histories) / 20), 1)
match_found = False
array_dividend = round(len(json_response_data['response']['exechistories'])/20)
if array_dividend == 0:
array_dividend == 1
for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]):
if (datetime.date.today() - datetime.timedelta(days=30) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
print("Found Date Match")
for index, item in enumerate(histories[::array_dividend]):
if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
match_found = True
break
checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend)
print(ct.colorText(f"{checkpoints_processed} checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue"))
checkpoint = item['checkpoint']
checkpoints_processed = round(len(histories) / array_dividend)
if ( index + 1 ) < checkpoints_processed:
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue"))
else:
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue"))
if match_found == True:
for index, item in enumerate(json_response_data['response']['exechistories']):
if index == len(json_response_data['response']['exechistories']) -1:
for index, item in enumerate(histories):
if index == len(histories) - 1:
print(ct.colorText(f"All Events Processed for {checkpoint}", "blue"))
checkpoint = item['checkpoint']
print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue"))
break
else:
if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
pass
else:
print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}")
json_output['response']['exechistories'].append(item)
match_found = False
json_output = json.dumps(json_output)
if outputjson == True:
return json_output
else: json_output['response']['exechistories'].append(item)
seen = {}
if os.path.exists(file_path):
with open(file_path, 'r') as file:
existing_data = json.load(file)
combined = existing_data['response']['exechistories'] + json_output['response']['exechistories']
else:
combined = json_output['response']['exechistories']
for item in combined:
key = (item.get('sha256'), item.get('filename'), item.get('hostname'))
seen[key] = item
deduplicated = list(seen.values())
with open(file_path, 'w') as file:
json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file)
json_output['response']['exechistories'].clear()
with open(file_path, 'r') as file:
final_output = json.load(file)
os.remove(file_path)
return json.dumps(final_output) if outputjson else None
def checkpoint_stomper(checkpoint, url, policy, headers):
json_output = {'error': 'Success', 'response': {'exechistories': []}}
@@ -94,4 +113,40 @@ def listPolicies(url):
policyids.append(list['groupid'])
choice = input(ct.colorText("Select Policy Group: ", "white"))
choice = int(choice) - 1
return choice, policiesnames
return choice, policiesnames, policyids
def listAllowlists(url):
endpoint = url + '/v1/application'
print(ct.colorText("[+] Grabbing All Allowlists", "cyan"))
payload = {}
headers = {
"X-APIKey": os.getenv('APIKEY')
}
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
parse_text = json.loads(response.text)
policiesnames = []
policyids = []
for index, list in enumerate(parse_text['response']['applications'], start=1):
if index >= 38:
print(ct.colorText(f"{index}. {list['name']}", "yellow"))
policiesnames.append(list['name'])
policyids.append(list['applicationid'])
choice = int(input(ct.colorText("Select allowlist: ", "white")))
if choice < 38:
print(ct.colorText("Please only choose an allowlist designed for this use - '38+'","red"))
elif choice >= 38:
choice = choice - 38
return choice, policiesnames, policyids
#Need else and catch for upper bound
def skipback(days):
"""
Generate a MongoDB ObjectId for a given number of days ago from today.
Adds 1 extra day to the input to look further back.
"""
adjusted_days = days + 1
date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days)
timestamp = int(date_days_ago.timestamp())
hex_timestamp = format(timestamp, '08x')
objectid_hex = hex_timestamp + '0000000000000000'
return ObjectId(objectid_hex)
utils/hashfunctions.py
+81 -23
View File
@@ -12,12 +12,12 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import pandas as pd
import requests
import os
import json
import utils.pretty as ct
def aggregateHashes(executions_json) -> pd.DataFrame:
"""
@@ -45,9 +45,13 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash,
aggregate applications and baselines into lists, then merges results back into agg_df to create a
"""
if 'sha256' not in agg_df.columns or agg_df.empty:
print("⚠️ 'sha256' column missing or DataFrame is empty. Skipping API query.")
return agg_df.copy() # Return as-is to avoid breaking downstream logic
endpoint = url + '/v1/hash/query'
payload = {
"hashes": agg_df['sha256'].tolist()
"hashes": agg_df['sha256'].tolist()
}
headers = {"X-APIKey": os.getenv('APIKEY')}
@@ -79,10 +83,23 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
df_api = pd.DataFrame(rows)
df = agg_df.merge(df_api, on="sha256", how="left")
aug_df = df[['sha256', 'filename_x', 'description', 'productname', 'productversion', 'publisher_y', 'publisher_x', 'netdomain', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline', 'reputation_lastseen', 'reputation_scannercount', 'reputation_scannermatch', 'reputation_status', 'reputation_threatlevel', 'reputation_threatname', 'reputation_timestamp']]
return aug_df
if 'sha256' not in df_api.columns:
print("⚠️ API response missing 'sha256'. Skipping merge.")
return agg_df.copy()
df = agg_df.merge(df_api, on="sha256", how="left")
# Only include columns that exist to avoid KeyErrors
expected_columns = ['sha256', 'filename_x', 'description', 'productname', 'productversion',
'publisher_y', 'publisher_x', 'netdomain', 'hostname', 'username',
'reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount',
'reputation_status', 'reputation_threatlevel', 'reputation_threatname',
'reputation_timestamp', 'pprocess', 'gprocess', 'commandline']
available_columns = [col for col in expected_columns if col in df.columns]
aug_df = df[available_columns]
return aug_df
def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
if untrusted_publishers is None:
@@ -93,29 +110,29 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
def reputationtool(row):
val = row["reputation_scannermatch"]
if pd.isna(val) or val == "N/A":
return row["publisher_y"] == "Not Signed"
return row["publisher"] == "Not Signed"
try:
return int(val) > threat_tolerance
except (ValueError, TypeError):
return row["publisher_y"] == "Not Signed"
return row["publisher"] == "Not Signed"
df["reputation_flag"] = df.apply(reputationtool, axis=1)
mask_needsreview = (
((df["publisher_y"] == "Not Signed") & df["reputation_flag"]) |
((df["publisher"] == "Not Signed") & df["reputation_flag"]) |
(df["reputation_status"] == "UNKNOWN")
)
mask_approved = (
(
(df["publisher_y"] != "Not Signed") &
~df["publisher_y"].isin(untrusted_publishers) &
(df["publisher"] != "Not Signed") &
~df["publisher"].isin(untrusted_publishers) &
~df["reputation_status"].isna()
) |
(
(df["publisher_y"] == "Not Signed") &
(df["publisher"] == "Not Signed") &
~df["reputation_flag"] &
~df["publisher_y"].isin(untrusted_publishers) &
~df["publisher"].isin(untrusted_publishers) &
~df["reputation_status"].isna()
)
)
@@ -126,18 +143,59 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
return needsreview_df, approved_df, unapproved_df
def destinationbuilder(df, df2):
# Step 1: Explode the 'sha256' list in df2 to create one row per sha256 value
df_expanded = df.explode('sha256')
def explode_and_deduplicate(df):
df['sha256'] = df['sha256'].str.split(',')
df = df.explode('sha256')
return df.drop_duplicates().reset_index(drop=True)
# Step 2: Create a new dataframe for the result
df_hashdestination = df_expanded.copy()
def clean_sha256(df, column='sha256'):
"""Discard quotes, brackets, and whitespace from sha256 values."""
df[column] = df[column].astype(str).str.strip("'[]\" ")
return df
# Step 3: Populate the 'Destination Allowlist' column based on comparison with df3
df_hashdestination['Destination Allowlist'] = df_hashdestination['sha256'].apply(
lambda x: 'Parent Policy Baseline' if x in df2['sha256'].values else "Destination Policy Allowlist"
def destinationHashes(
df_approved_paths: pd.DataFrame,
df_approved_hashes: pd.DataFrame,
df_hashes_auto_approved: pd.DataFrame,
df_hashes_manually_approved: pd.DataFrame,
):
# Deduplicate and explode all input DataFrames
df_approved_paths = explode_and_deduplicate(df_approved_paths)
df_approved_hashes = explode_and_deduplicate(df_approved_hashes)
df_hashes_auto_approved = explode_and_deduplicate(df_hashes_auto_approved)
df_hashes_manually_approved = explode_and_deduplicate(df_hashes_manually_approved)
# Clean sha256 values in all relevant DataFrames
df_approved_hashes = clean_sha256(df_approved_hashes)
df_hashes_auto_approved = clean_sha256(df_hashes_auto_approved)
df_hashes_manually_approved = clean_sha256(df_hashes_manually_approved)
# Create sets for faster lookup
auto_approved_sha256 = set(df_hashes_auto_approved['sha256'].values)
manually_approved_sha256 = set(df_hashes_manually_approved['sha256'].values)
# Debug: Print unmatched hashes
unmatched = set(df_approved_hashes['sha256']) - (auto_approved_sha256 | manually_approved_sha256)
print(f"Unmatched hashes: {unmatched}")
# Process df_approved_paths
df_paths = df_approved_paths.assign(destination='Path Exclusion')
df_paths = df_paths[['sha256', 'description', 'destination', 'grouped_directory', 'filename']]
# Process df_approved_hashes
df_hashes = df_approved_hashes.copy()
df_hashes['destination'] = df_hashes['sha256'].apply(
lambda x: 'Parent Policy Baseline' if x in auto_approved_sha256
else ('Child Policy Allowlist' if x in manually_approved_sha256 else None)
)
df_hashes = df_hashes.dropna(subset=['destination'])
df_hashes = df_hashes.assign(grouped_directory=None)
# Step 4: Return the new dataframe
# Use 'filename_x' only if it exists, otherwise fallback to 'filename'
filename_col = 'filename_x' if 'filename_x' in df_hashes.columns else 'filename'
selected_cols = ['sha256', 'description', 'destination', 'grouped_directory', filename_col]
df_hashes = df_hashes[selected_cols]
# Concatenate results
df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
return df_hashdestination
utils/pathfunctions.py
+114 -80
View File
@@ -16,103 +16,137 @@
import pandas as pd
import os
from itertools import chain
import ast
import re
def filepathInitialGroup(df: pd.DataFrame):
original_columns = df.columns.tolist()
def split_path(path):
parts = []
while True:
head, tail = os.path.split(path)
if tail:
parts.insert(0, tail)
path = head
else:
if head:
parts.insert(0, head)
break
return parts
# Step 1: Split comma-separated filepaths into lists
df["filename_x"] = df["filename_x"].str.split(",")
def local_common_pass(paths, min_parts=3):
results = {}
paths_sorted = sorted(paths)
for i, path in enumerate(paths_sorted):
candidates = []
# Step 2: Explode the list so each filepath becomes its own row
df = df.explode("filename_x", ignore_index=True)
if i > 0:
try:
candidates.append(os.path.commonpath([path, paths_sorted[i-1]]))
except ValueError:
# different drives, skip
pass
if i < len(paths_sorted) - 1:
try:
candidates.append(os.path.commonpath([path, paths_sorted[i+1]]))
except ValueError:
# different drives, skip
pass
# Step 3: Clean up whitespace and normalize paths
df["filename_x"] = df["filename_x"].str.strip()
df["filename_x"] = df["filename_x"].str.replace(r"\\\\", r"\\", regex=True)
df["filename_x"] = df["filename_x"].apply(lambda x: os.path.normpath(x) if pd.notna(x) else "")
best = path
best_len = 0
for c in candidates:
parts = split_path(c)
if len(parts) >= min_parts and len(parts) > best_len:
best = c
best_len = len(parts)
results[path] = best
return results
# Step 4: Extract directory and filename from each filepath
df["directory"] = df["filename_x"].apply(lambda x: os.path.normpath(os.path.dirname(x)) if pd.notna(x) else "")
df["filename"] = df["filename_x"].apply(lambda x: os.path.basename(x) if pd.notna(x) else "")
def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3):
dirs_series = df[col].astype(str).apply(os.path.dirname)
first_pass = local_common_pass(dirs_series.tolist(), min_parts)
second_pass = local_common_pass(list(first_pass.values()), min_parts)
df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]])
return df
# Step 5: Drop the original raw filepath column
df = df.drop(columns=["filename_x"])
def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant):
"""
Compute longest common paths, group filepaths, write CSV for review.
"""
df = df.drop_duplicates(subset=[col], keep='first')
df = add_longest_common_two_local(df, col=col, new_col=group_col)
grouped = df.groupby(group_col)[col].apply(list).reset_index()
grouped = grouped.sort_values(by=col)
print("Before filtering:", len(grouped))
grouped = grouped[grouped[col].apply(lambda x: len(x) >= min_number_in_group)]
filtered = grouped[grouped[group_col].apply(lambda x: len(os.path.normpath(x).split(os.sep)) >= path_length_constant)]
print("After filtering:", len(grouped))
# Helper functions for path manipulation
def get_parts(path):
return os.path.normpath(path).split(os.sep)
return filtered, df
def join_parts(parts):
return os.path.normpath(os.sep.join(parts))
def mask_from_csv(df, csv_path, filepath_col):
"""
Reads reviewed CSV of groups, keeps only files in approved groups.
"""
review_df = pd.read_csv(csv_path)
def longest_common_prefix(paths):
split_paths = [get_parts(p) for p in paths]
min_len = min(len(p) for p in split_paths)
prefix = []
for i in range(min_len):
current = split_paths[0][i]
if all(p[i] == current for p in split_paths):
prefix.append(current)
else:
break
return join_parts(prefix)
def parse_paths(val):
if isinstance(val, str):
try:
# Try to parse as a list
parsed = ast.literal_eval(val)
# If it's not a list, wrap it
return parsed if isinstance(parsed, list) else [parsed]
except (ValueError, SyntaxError):
# If parsing fails, treat it as a single path
return [val]
return [val]
# Step 6: Group directories by shared prefix
directories = df["directory"].tolist()
groups = []
used = set()
review_df[filepath_col] = review_df[filepath_col].apply(parse_paths)
for i, path in enumerate(directories):
if path in used:
continue
group = [path]
parts_i = get_parts(path)
# Flatten all approved file paths into a set for masking
approved_files = set()
for paths in review_df[filepath_col]:
approved_files.update(paths)
for j in range(i + 1, len(directories)):
parts_j = get_parts(directories[j])
common = os.path.commonprefix([parts_i, parts_j])
# Keep only rows in df that are in approved_files
masked_df = df[df[filepath_col].isin(approved_files)].copy()
remainder = df[~df[filepath_col].isin(approved_files)].copy()
return remainder
if (len(parts_i) > 3 and len(common) >= 3) or (len(parts_i) == 3 and len(common) >= 2):
group.append(directories[j])
used.add(directories[j])
elif len(common) == len(parts_i) - 1 and len(parts_i) > 3:
group.append(directories[j])
used.add(directories[j])
used.add(path)
groups.append(group)
def filter_and_drop(approved, eligiblepaths, min_hashes):
"""
Filters eligiblepaths to rows where all hashes are in approved,
then drops rows with fewer than min_hashes hashes.
"""
approved_hashes = set(approved['sha256'])
# Step 7: Map each original directory to its grouped prefix
prefix_map = {dir: longest_common_prefix(group) for group in groups for dir in group}
df["grouped_directory"] = df["directory"].map(prefix_map)
def all_hashes_approved(row):
return all(h in approved_hashes for h in row['sha256'])
# Step 8: Group the DataFrame by grouped_directory
aggregation = {col: (lambda x: list(x)) for col in original_columns if col not in ["filename_x"]}
aggregation.update({
"directory": lambda x: list(x),
"filename": lambda x: list(x)
})
filtered = eligiblepaths[eligiblepaths.apply(all_hashes_approved, axis=1)]
filtered = filtered[filtered['sha256'].apply(len) >= min_hashes]
grouped_df = df.groupby("grouped_directory", as_index=False).agg(aggregation)
return filtered
# Step 9: Split into eligible and ineligible paths based on depth
grouped_df["depth"] = grouped_df["grouped_directory"].apply(lambda x: len(get_parts(x)))
path_eligible = grouped_df[grouped_df["depth"] > 2].drop(columns=["depth"])
path_ineligible = grouped_df[grouped_df["depth"] <= 2].drop(columns=["depth"])
# Step 10: Move entries from eligible to ineligible if grouped_directory contains excluded directories
mask = path_eligible["grouped_directory"].str.contains(r"(?i)(?:\\Users|\\c\$\\Users|inetpub\\wwwroot|windows\\temp)", na=False)
move_to_ineligible = path_eligible[mask]
path_eligible = path_eligible[~mask]
path_ineligible = pd.concat([path_ineligible, move_to_ineligible], ignore_index=True)
# Step 11: Deduplicate list elements in all columns
def deduplicate_lists(df):
for col in df.columns:
if df[col].apply(lambda x: isinstance(x, list)).all():
df[col] = df[col].apply(lambda x: list({str(item): item for item in chain.from_iterable(x if isinstance(x[0], list) else [x])}.values()))
def inspect_parquet(path):
try:
df = pd.read_parquet(path)
print(f"✅ Successfully read: {path}")
print(f"📄 Columns: {df.columns.tolist()}")
print(f"🔢 Rows: {len(df)}")
return df
except Exception as e:
print(f"❌ Error reading {path}: {e}")
return pd.DataFrame()
path_eligible = deduplicate_lists(path_eligible)
path_ineligible = deduplicate_lists(path_ineligible)
return path_eligible, path_ineligible
def regulator(paths, case_insensitive=True):
"""
Build a Python raw string regex that matches any of the given Windows path fragments.
"""
escaped = [re.escape(p) for p in paths]
pattern = "(?:" + "|".join(escaped) + ")"
if case_insensitive:
pattern = pattern
print(f"Regulator is providing {pattern}")
return f'r"{pattern}"'
utils/policyfunctions.py
+64
View File
@@ -0,0 +1,64 @@
# Copyright (C) 2025 James Brotosky, Brandon Wickline
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import requests
import json
import os
import utils.pretty as ct
def addHash(policy, hash):
print(f"Adding the following hashes to {policy}:")
print(hash)
def addPath(policy, hash):
print(f"Adding the following Path Exclusions to {policy}:")
print(hash)
def addHashReal(url, allowlistID, hashlist):
endpoint = url + '/v1/hash/application/add'
print(ct.colorText("[+] Grabbing All Categories", "cyan"))
payload = {
"applicationid" : allowlistID,
"hashes" : hashlist
}
headers = {
"X-APIKey": os.getenv('APIKEY')
}
try:
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
response.raise_for_status() # Raise an error for bad status codes
parse_text = json.loads(response.text)
print(parse_text)
except requests.exceptions.RequestException as e:
return {"error": str(e)}
def addPathReal(url, grouplistID, pathlist):
endpoint = url + '/v1/group/path/add'
print(ct.colorText("[+] Grabbing All Categories", "cyan"))
payload = {
"applicationid" : grouplistID,
"hashes" : pathlist
}
headers = {
"X-APIKey": os.getenv('APIKEY')
}
try:
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
response.raise_for_status() # Raise an error for bad status codes
parse_text = json.loads(response.text)
print(parse_text)
except requests.exceptions.RequestException as e:
return {"error": str(e)}
utils/pretty.py
+75
View File
@@ -136,3 +136,78 @@ def style_dataframe_dark(df, output_html_path=None, overwrite=True):
print(f"✅ Styled table saved to temporary file: {temp_path}")
else:
return styled_html
def displayIntro():
print(colorText(r"""
███
████ ░████████
█████████████ ███████████████
█████████████████████ █████████████████████
███████████████████ ██████████████████████▓
███████████████████ ██████████████████████
█████████████████████ ███████████████████████
████████████████████████████████████████████████████████
█████████ ██ ██ █████████
█████████ ██ ███ █ █████████
█████████ ██ ████ █████ █████████████
█████████ ██ ██████ █████████████
████████ ██ ███████ ████████████░
███████ ██ ██▓ ██████ ████████████
██████ ██ ████ █████ ███████████
█████████████████████████████████████████████████
▒████████████████████ ██████████████████
███████████████████ ███████████████▒
███████████████ █████████████
██████████ ███████████
████████
████
""", "yellow"))
print(colorText(r"""
_____ .__ .__ __ ___________ .__
/ _ \ |__|______| | ____ ____ | | __ \__ ___/___ ____ | | ______
/ /_\ \| \_ __ \ | / _ \_/ ___\| |/ / | | / _ \ / _ \| | / ___/
/ | \ || | \/ |_( <_> ) \___| < | |( <_> | <_> ) |__\___ \
\____|__ /__||__| |____/\____/ \___ >__|_ \ |____| \____/ \____/|____/____ >
\/ \/ \/ \/
""", "cyan"))
print(colorText("=================================================================================", "cyan"))
print(colorText("======================== Welcome to the Airlock API Tool ========================", "cyan"))
print(colorText("=================================================================================", "cyan"))
def areYouSure():
print(colorText(f"*******************************************************************************************************************************************","red"))
print(colorText(f"*=========================================================================================================================================*","yellow"))
print(colorText(f"*=========================================================================================================================================*","red"))
print(colorText(f"*-------------This program will now begin to make changes to the Airlock Console. Do you understand and agree to proceed? ----------------*", "yellow"))
print(colorText(f"*=========================================================================================================================================*","red"))
print(colorText(f"*=========================================================================================================================================*","yellow"))
print(colorText(f"*******************************************************************************************************************************************","red"))
def locked():
print(colorText(r"""
████████████████████████████████████████████████████████████████
███ ██
██ ██████ ███
██ ████████████ ███
██ ████ ███ ███
██ ███ ███ ███
██ ███ ███ ███
██ ▒████████████████████ ███
██ ██████████████████████ ███
██ ██████████████████████ ███
██ ██████████████████████ ███
██ ██████████████████████ ███
██ ██████████████████████ ███
██ ███
███ ███
████████████████████████████████████████████████████████████████████
▒██████████████████████████████████████████████████████████████████▒
▒████
▒████
▓██████████████████████████████████████████
█████████████████████████████████████████████░
""", "yellow"))