mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Major Refactor now allows multiple policies to be selected

Browse Source
This commit is contained in:
Zarithas
2025-09-26 14:57:12 -04:00
parent 74b3f7b2e1
commit 0a26ed71c1
8 changed files with 777 additions and 1133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/policyfunctions.py
+264 -59
View File
@@ -14,7 +14,11 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#Local Imports
import utils.pretty as ct
import utils.utils as ct
import utils.hashfunctions as hashf
import utils.pathfunctions as pathf
import utils.policyfunctions as policyf
#Standard Libary Imports:
import datetime
import gc
@@ -44,7 +48,6 @@ def addPub(url, policy, publist):
print(f"Adding the following Publishers to {policy}:")
for p in publist:
print(p)
def addHashReal(url, allowlistID, hashlist):
endpoint = url + '/v1/hash/application/add'
@@ -62,7 +65,6 @@ def addHashReal(url, allowlistID, hashlist):
parse_text = json.loads(response.text)
print(parse_text)
def addPathReal(url, grouplistID, pathlist):
endpoint = url + '/v1/group/path/add'
payload = {
@@ -77,7 +79,6 @@ def addPathReal(url, grouplistID, pathlist):
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
print(response.text)
def addPubReal(url, grouplistID, publist):
endpoint = url + '/v1/group/publisher/add'
payload = {
@@ -92,27 +93,29 @@ def addPubReal(url, grouplistID, publist):
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
print(response.text)
def getPolicyInfo(url, policy, type, days, parquet=True):
executionhist_policy = pd.DataFrame()
exehist = pullPolicyExechistories(url, policy, type, days, True)
data = json.loads(exehist)
executionhist_policy = pd.DataFrame(data["response"]["exechistories"])
if not executionhist_policy.empty:
executionhist_policy = executionhist_policy[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
executionhist_policy = executionhist_policy.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
executionhist_policy = executionhist_policy.sort_values(by=['sha256', 'filename'])
if parquet: executionhist_policy.to_parquet(f"parquet\\execution_history_{policy}.parquet", index=False)
print(ct.colorText(f"Staging of Execution history for policy: {policy} is complete", "green"))
del data
del exehist
gc.collect()
if exehist is not None:
data = json.loads(exehist)
executionhist_policy = pd.DataFrame(data["response"]["exechistories"])
if not executionhist_policy.empty:
executionhist_policy = executionhist_policy[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
executionhist_policy['policy'] = policy # Add policy column here
executionhist_policy = executionhist_policy.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
executionhist_policy = executionhist_policy.sort_values(by=['sha256', 'filename'])
if parquet:
executionhist_policy.to_parquet(f"prepare_policy\\parquet\\execution_history_{policy}.parquet", index=False)
print(ct.colorText(f"Staging of Execution history for policy: {policy} is complete", "green"))
del data
del exehist
gc.collect()
return executionhist_policy
def sendToPolicy(url, first_policy, second_policy, destination_name, destination_id, allowlist_parent_name, allowlist_parent_id, allowlist_child_name, allowlist_child_id):
pathexclusions = pd.read_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet")
allowbyhash = pd.read_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet")
publishers = pd.read_parquet(f"parquet\\publishers_{first_policy}_{second_policy}.parquet")
def sendToPolicy(url, paths, hashes, publishers, destination_name, destination_id, allowlist_name, allowlist_id):
pathexclusions = pd.read_parquet(paths)
allowbyhash = pd.read_parquet(hashes)
publishers = ct.tryToReadCSV(publishers)
ct.areYouSure()
confirmation = input(ct.colorText("Type 'I AGREE' to continue: ","white"))
@@ -137,22 +140,22 @@ def sendToPolicy(url, first_policy, second_policy, destination_name, destination
addPathReal(url, destination_id,processed_paths)
print(ct.colorText(f"Adding publishers to {destination_name}", "yellow"))
if publishers.empty:
print(ct.colorText("The publishers list is empty.", "red"))
else:
publisher_list = publishers['publisher'].tolist()
addPubReal(url, destination_id, publisher_list)
publisher_list = publishers['publisher'].tolist()
addPubReal(url, destination_id, publisher_list)
print(ct.colorText(f"Adding hashes to {allowlist_parent_name}", "yellow"))
print(ct.colorText(f"These hashes would be added to {allowlist_name}", "yellow"))
allowlist_parenthashlist = allowbyhash[allowbyhash['reputation_status'] == 'KNOWN']['sha256'].unique().tolist()
addHashReal(url, allowlist_parent_id,allowlist_parenthashlist)
print(ct.colorText(f"Adding hashes to {allowlist_child_name}", "yellow"))
allowlist_childhashlist = allowbyhash[allowbyhash['reputation_status'] == 'UNKNOWN']['sha256'].unique().tolist()
addHashReal(url, allowlist_child_id, allowlist_childhashlist)
allowlist = allowbyhash['sha256'].unique().tolist()
addHash(url, allowlist_id,allowlist)
ct.locked()
exit()
exit()
else:
print(ct.colorText("Operation aborted. You MUST EXPLICITLY AGREE to proceed.", "red"))
@@ -281,8 +284,7 @@ def listATPolicies(url):
print(ct.colorText(f"[!] Failed to parse response: {e}", "red"))
return {}
def listAllowlists(url):
def listAllowlists(url: str) -> tuple[int, list, list]:
endpoint = url + '/v1/application'
print(ct.colorText("[+] Grabbing All Allowlists", "cyan"))
payload = {}
@@ -293,18 +295,23 @@ def listAllowlists(url):
parse_text = json.loads(response.text)
policiesnames = []
policyids = []
for index, list in enumerate(parse_text['response']['applications'], start=1):
for index, item in enumerate(parse_text['response']['applications'], start=1):
if index >= 38:
print(ct.colorText(f"{index}. {list['name']}", "yellow"))
policiesnames.append(list['name'])
policyids.append(list['applicationid'])
choice = int(input(ct.colorText("Select allowlist: ", "white")))
if choice < 38:
print(ct.colorText("Please only choose an allowlist designed for this use - '38+'","red"))
elif choice >= 38:
choice = choice - 38
return choice, policiesnames, policyids
#Need else and catch for upper bound
print(ct.colorText(f"{index}. {item['name']}", "yellow"))
policiesnames.append(item['name'])
policyids.append(item['applicationid'])
while True:
try:
choice = int(input(ct.colorText("Select allowlist: ", "white")))
if choice < 38 or choice > len(parse_text['response']['applications']):
print(ct.colorText("Please only choose an allowlist designed for this use - '38+'", "red"))
else:
adjusted_choice = choice - 38
return adjusted_choice, policiesnames, policyids
except ValueError:
print(ct.colorText("Invalid input. Please enter a number.", "red"))
def skipback(days):
"""
@@ -318,10 +325,10 @@ def skipback(days):
objectid_hex = hex_timestamp + '0000000000000000'
return ObjectId(objectid_hex)
def sendToPolicyTest(url, first_policy, second_policy, destination_name, destination_id, allowlist_parent_name, allowlist_parent_id, allowlist_child_name, allowlist_child_id):
pathexclusions = pd.read_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet")
allowbyhash = pd.read_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet")
publishers = pd.read_parquet(f"parquet\\publishers_{first_policy}_{second_policy}.parquet")
def sendToPolicyTest(url, paths, hashes, publishers, destination_name, destination_id, allowlist_id, allowlist_name):
pathexclusions = pd.read_parquet(paths)
allowbyhash = pd.read_parquet(hashes)
publishers = ct.tryToReadCSV(publishers)
print(ct.colorText(f"These path exclusions would be added to {destination_name}", "yellow"))
@@ -342,20 +349,17 @@ def sendToPolicyTest(url, first_policy, second_policy, destination_name, destina
print(ct.colorText(f"These publishers would added to {destination_name}", "yellow"))
publisher_list = publishers['publisher'].tolist()
addPub(url, destination_id, publisher_list)
if publishers.empty:
print(ct.colorText("The publishers list is empty.", "red"))
else:
publisher_list = publishers['publisher'].tolist()
addPub(url, destination_id, publisher_list)
print(ct.colorText(f"These hashes would be added to {allowlist_parent_name}", "yellow"))
print(ct.colorText(f"These hashes would be added to {allowlist_name}", "yellow"))
allowlist_parenthashlist = allowbyhash[allowbyhash['reputation_status'] == 'KNOWN']['sha256'].unique().tolist()
addHash(url, allowlist_parent_id,allowlist_parenthashlist)
allowlist = allowbyhash['sha256'].unique().tolist()
addHash(url, allowlist_id,allowlist)
print(ct.colorText(f"These hashes would be added to {allowlist_child_name}", "yellow"))
allowlist_childhashlist = allowbyhash[allowbyhash['reputation_status'] == 'UNKNOWN']['sha256'].unique().tolist()
addHash(url, allowlist_child_id, allowlist_childhashlist)
def updateAuditPoliciesFromEnforcementPolices(url, policy_relationship_map):
for enforcement_policy, audit_policy in policy_relationship_map.items():
assignPoliciesfromGroup(url, enforcement_policy, audit_policy)
@@ -376,7 +380,6 @@ def assignPoliciesfromGroup(url, source_policy_id, target_policy_id):
parse_text = json.loads(response.text)
print(parse_text)
def turnOnAudit(url, policyid):
endpoint = url + '/v1/group/settings/auditmode'
@@ -404,3 +407,205 @@ def agentsInPolicy(url, policyid):
response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
parse_text = json.loads(response.text)
print(parse_text)
def prepare_to_enforce(url, bad_publisher_list, pups, badpathparts, threat_tolerance_constant = 4, path_exclusion_constant = 4, min_files_for_path = 4):
destination_name = " "
destination_id = " "
allowlist_name = " "
allowlist_id = " "
policylist = []
type = [1, 2, 6, 7]
parq_base_dir = "prepare_policy\\parquet\\"
appr_base_dir = "prepare_policy\\approved\\"
needappr_base_dir = "prepare_policy\\needs_approved\\"
pflight_base_dir = "prepare_policy\\preflight\\"
#If the directorys where we're going to store our output dont exist, make them.
os.makedirs(parq_base_dir, exist_ok=True)
os.makedirs(needappr_base_dir, exist_ok=True)
os.makedirs(appr_base_dir, exist_ok=True)
os.makedirs(pflight_base_dir, exist_ok=True)
while True:
ct.printEnforceChecklist(parq_base_dir,appr_base_dir, needappr_base_dir, pflight_base_dir, policylist, allowlist_name, destination_name)
choice = input(ct.colorText("\nEnter your choice: ", "white"))
if choice == "1":
while True:
choice, policynames, policyid = listPolicies(url)
selected_policy = policynames[choice]
if selected_policy not in policylist:
policylist.append(selected_policy)
while True:
answer = input(ct.colorText("Do you want to load another policy? (yes/no): ", "white")).strip().lower()
if answer in ("no", "n"):
break # Exit the inner loop and then the outer loop
elif answer in ("yes", "y"):
break # Exit the inner loop and continue the outer loop
else:
print(ct.colorText("Please answer with 'yes' or 'no'.", "red"))
if answer in ("no", "n"):
break
print(policylist)
elif choice == "2":
print(ct.colorText(f"Please choose destination_name Policy for Path Exclusions","white"))
choice, policynames, policyid = listPolicies(url)
#print(allowlist_parent_tuple)
destination_name = policynames[choice]
destination_id = policyid[choice]
print(ct.colorText(f"Please choose Allowlist for Hashes","white"))
choice, allowlists,allowid = listAllowlists(url)
#print(allowlist_parent_tuple)
allowlist_name = allowlists[choice]
allowlist_id = allowid[choice]
print(destination_name, allowlist_name)
elif choice == "3":
policyf.buildExecHistory(url,
policylist,
parq_base_dir,
needappr_base_dir,
type,
threat_tolerance_constant,
bad_publisher_list,
pups,
)
csvs = [f"{needappr_base_dir}unknown_hashes.csv", f"{needappr_base_dir}good_hashes.csv"]
#Since we want to build paths as if they were all in the same policy to begin with, lets group them that way
for csv in csvs:
df = ct.tryToReadCSV(csv)
df['policy'] = destination_name
df.to_csv(csv)
elif choice == "4":
pathf.generateApprovables(needappr_base_dir, appr_base_dir, parq_base_dir, badpathparts, bad_publisher_list, path_exclusion_constant, min_files_for_path)
elif choice == "5":
savePreFlights(appr_base_dir, parq_base_dir, pflight_base_dir)
elif choice == "6":
if os.path.exists(f"{pflight_base_dir}final_path_exclusions.html") and os.path.exists(f"{pflight_base_dir}\\final_hash_approvals.html") and allowlist_name != " " and destination_name != " ":
sendToPolicyTest(
url,
f"{parq_base_dir}final_path_exclusions.parquet",
f"{parq_base_dir}final_hash_approvals.parquet",
f"{appr_base_dir}publishers.parquet",
destination_name,
destination_id,
allowlist_name,
allowlist_id
)
elif choice == "7":
if os.path.exists(f"{pflight_base_dir}final_path_exclusions.html") and os.path.exists(f"{pflight_base_dir}\\final_hash_approvals.html") and allowlist_name != " " and destination_name != " ":
sendToPolicy(
url,
f"{parq_base_dir}final_path_exclusions.parquet",
f"{parq_base_dir}final_hash_approvals.parquet",
f"{appr_base_dir}publishers.parquet",
destination_name,
destination_id,
allowlist_name,
allowlist_id,
)
elif choice == "R":
pathf.clean_folders(parq_base_dir, appr_base_dir, needappr_base_dir, pflight_base_dir)
elif choice == "Q":
break
else:
print(ct.colorText("Invalid choice. Please try again.", "red"))
def buildExecHistory(url,
policylist,
parq_base_dir,
needappr_base_dir,
type,
threat_tolerance_constant,
bad_publisher_list,
pups
):
exe_hist_parq_list = []
while True:
try:
history_days = int(input("Enter the how many days in of history do you want to pull - select a number between 1 and 150: "))
if 1 <= history_days <= 150:
break
else:
print("Invalid input. Please enter a number between 1 and 150.")
except ValueError:
print("Invalid input. Please enter a valid integer.")
for policy in policylist:
policy_exec_history = policyf.getPolicyInfo(url, policy, type, history_days)
policy_exec_history['policy'] = policy
policy_exec_history.to_parquet(f"{parq_base_dir}Exec_Hist_{policy}.parquet")
exe_hist_parq_list.append(f"{parq_base_dir}Exec_Hist_{policy}.parquet")
augmented_hashlist = hashf.combineHashes(url, exe_hist_parq_list)
augmented_hashlist.to_parquet(f"{parq_base_dir}augmentedHashlist.parquet",index=False)
needsreview_df, approved_df, unapproved_df = hashf.categorizeHashes(augmented_hashlist, threat_tolerance_constant, bad_publisher_list, pups)
needsreview_df.to_parquet(f"{parq_base_dir}needsreview.parquet",index=False)
approved_df.to_parquet(f"{parq_base_dir}approved.parquet",index=False)
unapproved_df.to_parquet(f"{parq_base_dir}unapproved.parquet",index=False)
condensed_executions = hashf.condenseExecutions(exe_hist_parq_list)
condensed_executions.to_parquet(f"{parq_base_dir}condensed_executions.parquet", index=False)
unknown, good, bad = hashf.divideSortedHashExecutions(
f"{parq_base_dir}needsreview.parquet",
f"{parq_base_dir}approved.parquet",
f"{parq_base_dir}unapproved.parquet",
f"{parq_base_dir}condensed_executions.parquet",
pups
)
dataframes = {
"unknown_hashes" : unknown,
"good_hashes": good,
"bad_hashes": bad
}
for name, df in dataframes.items():
df.to_csv(f"{needappr_base_dir}{name}.csv", index=False)
df.to_parquet(f"{parq_base_dir}{name}.parquet", index=False)
ct.style_dataframe_dark(df, f"{needappr_base_dir}{name}.html")
def savePreFlights(appr_base_dir, parq_base_dir, pflight_base_dir):
if os.path.exists(f"{appr_base_dir}primary_Paths.csv"):
if not os.path.exists(f"{parq_base_dir}final_hash_approvals.parquet") and not os.path.exists(f"{parq_base_dir}final_path_exclusions.parquet"):
pathexclusions, allowbyhash = hashf.generatePreflights(
f"{parq_base_dir}all_hashes.parquet",
f"{appr_base_dir}primary_Paths.csv",
f"{appr_base_dir}secondary_Paths.csv")
dataframes = {
"final_path_exclusions" : pathexclusions,
"final_hash_approvals": allowbyhash
}
for name, df in dataframes.items():
df.to_csv(f"{pflight_base_dir}{name}.csv", index=False)
df.to_parquet(f"{parq_base_dir}{name}.parquet", index=False)
ct.style_dataframe_dark(df, f"{pflight_base_dir}{name}.html")