mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Working Chunkinantor

Browse Source
This commit is contained in:
brotoskyj
2025-09-01 13:16:43 -04:00
parent e9c7ba132a
commit 4b291a9521
2 changed files with 38 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AirlockTools.py
+9 -9
View File
@@ -291,7 +291,7 @@ def menu_prepare_to_enforce():
executionhist_policy1 = executionhist_policy1.sort_values(by=['sha256','filename']) executionhist_policy1 = executionhist_policy1.sort_values(by=['sha256','filename'])
executionhist_policy1.to_csv(f"dataframe_csv\\executionhist_{first_policy}.csv", index=False) executionhist_policy1.to_csv(f"dataframe_csv\\executionhist_{first_policy}.csv", index=False)
ct.style_dataframe_dark(executionhist_policy1, f"dataframe_html\\executionhist_{first_policy}.html") #ct.style_dataframe_dark(executionhist_policy1, f"dataframe_html\\executionhist_{first_policy}.html")
print(ct.colorText(f"Staging of Execution history for policy: {first_policy} is complete","green")) print(ct.colorText(f"Staging of Execution history for policy: {first_policy} is complete","green"))
if not os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"): if not os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"):
@@ -303,7 +303,7 @@ def menu_prepare_to_enforce():
executionhist_policy2 = executionhist_policy2.sort_values(by=['sha256','filename']) executionhist_policy2 = executionhist_policy2.sort_values(by=['sha256','filename'])
executionhist_policy2.to_csv(f"dataframe_csv\\executionhist_{second_policy}.csv", index=False) executionhist_policy2.to_csv(f"dataframe_csv\\executionhist_{second_policy}.csv", index=False)
ct.style_dataframe_dark(executionhist_policy2, f"dataframe_html\\executionhist_{second_policy}.html") #ct.style_dataframe_dark(executionhist_policy2, f"dataframe_html\\executionhist_{second_policy}.html")
print(ct.colorText(f"Staging of Exection history for policy: {first_policy} is complete","green")) print(ct.colorText(f"Staging of Exection history for policy: {first_policy} is complete","green"))
#Combine the two policies execution histories #Combine the two policies execution histories
@@ -311,20 +311,20 @@ def menu_prepare_to_enforce():
if second_policy is first_policy: if second_policy is first_policy:
execuctionhist_combined = executionhist_policy1 execuctionhist_combined = executionhist_policy1
execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False) execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html") #ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Dataframes have been combined","green")) print(ct.colorText(f"Dataframes have been combined","green"))
elif os.path.exists(f"dataframe_csv\\executionhist_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"): elif os.path.exists(f"dataframe_csv\\executionhist_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"):
execuctionhist_combined = pd.concat([tryToReadCSV(f"dataframe_csv\\executionhist_{first_policy}.csv") , tryToReadCSV(f"dataframe_csv\\executionhist_{second_policy}.csv")], ignore_index=True).sort_values(by=['sha256','filename']) execuctionhist_combined = pd.concat([tryToReadCSV(f"dataframe_csv\\executionhist_{first_policy}.csv") , tryToReadCSV(f"dataframe_csv\\executionhist_{second_policy}.csv")], ignore_index=True).sort_values(by=['sha256','filename'])
execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False) execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html") #ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Dataframes have been combined","green")) print(ct.colorText(f"Dataframes have been combined","green"))
#Keep only unique combinations of hash, filename, and hostname #Keep only unique combinations of hash, filename, and hostname
if f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv": if f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv":
unique_executions = tryToReadCSV(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv").drop_duplicates(subset=['sha256', 'filename', 'hostname']) unique_executions = tryToReadCSV(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv").drop_duplicates(subset=['sha256', 'filename', 'hostname'])
unique_executions.to_csv(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv") unique_executions.to_csv(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv")
ct.style_dataframe_dark(unique_executions, f"dataframe_html\\unique_execuctions.html") #ct.style_dataframe_dark(unique_executions, f"dataframe_html\\unique_execuctions.html")
#Add Hash info to the combined execution history #Add Hash info to the combined execution history
@@ -332,7 +332,7 @@ def menu_prepare_to_enforce():
print(ct.colorText(f"Preparing to pull hash info","green")) print(ct.colorText(f"Preparing to pull hash info","green"))
augmented_combo= utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv")) augmented_combo= utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv"))
augmented_combo.to_csv(f"dataframe_csv\\augmented_combo_{first_policy}_{second_policy}.csv", index=False) augmented_combo.to_csv(f"dataframe_csv\\augmented_combo_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(augmented_combo, f"dataframe_html\\augmented_combo_{first_policy}_{second_policy}.html") #ct.style_dataframe_dark(augmented_combo, f"dataframe_html\\augmented_combo_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Hash reputation info added to dataframe","green")) print(ct.colorText(f"Hash reputation info added to dataframe","green"))
#Categorize the hashes #Categorize the hashes
@@ -345,19 +345,19 @@ def menu_prepare_to_enforce():
categorized[0].sort_values(by=['sha256','filename_x']) categorized[0].sort_values(by=['sha256','filename_x'])
categorized[0].to_csv(f"dataframe_csv\\hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False) categorized[0].to_csv(f"dataframe_csv\\hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(categorized[0], f"dataframe_html\\hashes_needing_approval_{first_policy}_{second_policy}.html") #ct.style_dataframe_dark(categorized[0], f"dataframe_html\\hashes_needing_approval_{first_policy}_{second_policy}.html")
if not categorized[1].empty: if not categorized[1].empty:
categorized[1].sort_values(by=['sha256','filename_x']) categorized[1].sort_values(by=['sha256','filename_x'])
categorized[1].to_csv(f"dataframe_csv\\automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False) categorized[1].to_csv(f"dataframe_csv\\automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(categorized[1], f"dataframe_html\\automatically_approved_hashes_{first_policy}_{second_policy}.html") #ct.style_dataframe_dark(categorized[1], f"dataframe_html\\automatically_approved_hashes_{first_policy}_{second_policy}.html")
if not categorized[2].empty: if not categorized[2].empty:
categorized[2].sort_values(by=['sha256','filename_x']) categorized[2].sort_values(by=['sha256','filename_x'])
categorized[2].to_csv(f"dataframe_csv\\unapproved_hashes__{first_policy}_{second_policy}.csv", index=False) categorized[2].to_csv(f"dataframe_csv\\unapproved_hashes__{first_policy}_{second_policy}.csv", index=False)
ct.style_dataframe_dark(categorized[2], f"dataframe_html\\unapproved_hashes_{first_policy}_{second_policy}.html") #ct.style_dataframe_dark(categorized[2], f"dataframe_html\\unapproved_hashes_{first_policy}_{second_policy}.html")
print(ct.colorText(f"Hashes have been categorized","green")) print(ct.colorText(f"Hashes have been categorized","green"))
utils/allowlist.py
+29 -10
View File
@@ -22,9 +22,7 @@ import utils.pretty as ct
import ijson import ijson
def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool):
if not os.path.exists('chunkinator.csv'): file_path = 'chunkinator.json'
with open('chunkinator.csv', 'x') as file:
pass
headers = { headers = {
"X-APIKey": os.getenv('APIKEY') "X-APIKey": os.getenv('APIKEY')
} }
@@ -39,27 +37,48 @@ def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool):
if array_dividend == 0: if array_dividend == 0:
array_dividend == 1 array_dividend == 1
for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]): for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]):
if (datetime.date.today() - datetime.timedelta(days=30) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): if (datetime.date.today() - datetime.timedelta(days=60) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
print("Found Date Match")
match_found = True match_found = True
break break
checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend) checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend)
print(ct.colorText(f"{checkpoints_processed} checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) if (index + 1) < checkpoints_processed:
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed and Found with Date Match. Last Checkpoint: {item['checkpoint']}.", "blue"))
else:
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed. Last Checkpoint: {item['checkpoint']}.", "blue"))
checkpoint = item['checkpoint'] checkpoint = item['checkpoint']
if match_found == True: if match_found == True:
for index, item in enumerate(json_response_data['response']['exechistories']): for index, item in enumerate(json_response_data['response']['exechistories']):
if index == len(json_response_data['response']['exechistories']) -1: if index == len(json_response_data['response']['exechistories']) -1:
checkpoint = item['checkpoint'] checkpoint = item['checkpoint']
print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) print(ct.colorText(f"All Events Processed. Next Checkpoint: {item['checkpoint']}", "blue"))
break break
else: else:
if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): if (datetime.date.today() - datetime.timedelta(days=60) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
pass pass
else: else:
print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}") #print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}")
json_output['response']['exechistories'].append(item) json_output['response']['exechistories'].append(item)
match_found = False if os.path.exists(file_path):
seen = {}
with open(file_path, 'r') as file:
temp_file = json.load(file)
combined_data = temp_file['response']['exechistories'] + json_output['response']['exechistories']
for item in combined_data:
key = (item.get('sha256'), item.get('filename'), item.get('hostname'))
seen[key] = item
deduplicated_data = list(seen.values())
with open(file_path, 'w') as file:
json.dump({'error': 'Success', 'response': {'exechistories': deduplicated_data}}, file)
json_output = {'error': 'Success', 'response': {'exechistories': []}}
else:
with open(file_path, 'a') as file:
json.dump(json_output, file)
json_output = {'error': 'Success', 'response': {'exechistories': []}}
with open(file_path, 'r') as file:
json_output = json.load(file)
json_output = json.dumps(json_output) json_output = json.dumps(json_output)
os.remove(file_path)
if outputjson == True: if outputjson == True:
return json_output return json_output