mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge branch 'MemOpt' of https://git.racooncity.org/brotoskyj/Airlocktools into MemOpt

Browse Source
This commit is contained in:
=
2025-09-03 17:23:06 -04:00
parent b52ce86599 3f2717d972
commit e35e6118e7
3 changed files with 58 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AirlockTools.py
+13
View File
@@ -433,9 +433,22 @@ def menu_prepare_to_enforce():
allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False) allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False)
<<<<<<< HEAD
allowbyhash.sort_values(by=["filename"]) allowbyhash.sort_values(by=["filename"])
ct.style_dataframe_dark(allowbyhash, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html") ct.style_dataframe_dark(allowbyhash, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
=======
easyview = allowbyhash.groupby('sha256').agg(list).reset_index()
# Deduplicate and sort by first item in list
for col in easyview.columns:
if col != 'sha256':
easyview[col] = easyview[col].apply(lambda x: list(set(x)))
if col in ["reputation_status", "filename"]:
# Sort the list to ensure consistent first item
easyview[col] = easyview[col].apply(lambda x: sorted(x)[0] if x else None)
ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
>>>>>>> 3f2717d972d811b0615f9b250aeb5191b255aa47
ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html") ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html")
requirements.txt
+1
View File
@@ -2,3 +2,4 @@ pandas==2.3.2
python-dotenv==1.1.1 python-dotenv==1.1.1
Requests==2.32.5 Requests==2.32.5
urllib3==2.5.0 urllib3==2.5.0
tqdm
utils/allowlist.py
+44 -41
View File
@@ -21,6 +21,8 @@ import ijson
import os import os
from bson import ObjectId from bson import ObjectId
import datetime import datetime
import tqdm
import sys
def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
file_path = 'chunkinator.json' file_path = 'chunkinator.json'
@@ -33,46 +35,47 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
headers = {"X-APIKey": os.getenv('APIKEY')} headers = {"X-APIKey": os.getenv('APIKEY')}
checkpoint = str(skipback(days)) checkpoint = str(skipback(days))
json_output = {'error': 'Success', 'response': {'exechistories': []}} json_output = {'error': 'Success', 'response': {'exechistories': []}}
while True: with tqdm.tqdm(file=sys.stdout, leave=True, total=10000, desc=f"Checkpoint Progess: {checkpoint}", colour="blue", initial=1) as filebar:
json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) with tqdm.tqdm(file=sys.stdout, leave=True, total=100, desc=f"Total of {policiesnames} Complete: ") as pbar:
histories = json_response_data['response']['exechistories'] while True:
if not histories: json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers)
break histories = json_response_data['response']['exechistories']
array_dividend = max(round(len(histories) / 20), 1) filebar.total=len(histories)
match_found = False if not histories:
for index, item in enumerate(histories[::array_dividend]): break
if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): match_found = True
match_found = True if match_found == True:
break for index, item in enumerate(histories):
checkpoints_processed = round(len(histories) / array_dividend) if index == len(histories) - 1:
if ( index + 1 ) < checkpoints_processed: checkpoint = item['checkpoint']
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue")) filebar.desc = f"Checkpoint Progress: {checkpoint}"
else: break
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue")) else:
if match_found == True: if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
for index, item in enumerate(histories): pass
if index == len(histories) - 1: else: json_output['response']['exechistories'].append(item)
print(ct.colorText(f"All Events Processed for {checkpoint}", "blue")) filebar.update(1)
checkpoint = item['checkpoint'] filebar.refresh()
break seen = {}
else: if os.path.exists(file_path):
if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): with open(file_path, 'r') as file:
pass existing_data = json.load(file)
else: json_output['response']['exechistories'].append(item) combined = existing_data['response']['exechistories'] + json_output['response']['exechistories']
seen = {} else:
if os.path.exists(file_path): combined = json_output['response']['exechistories']
with open(file_path, 'r') as file: for item in combined:
existing_data = json.load(file) key = (item.get('sha256'), item.get('filename'), item.get('hostname'))
combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] seen[key] = item
else: deduplicated = list(seen.values())
combined = json_output['response']['exechistories'] with open(file_path, 'w') as file:
for item in combined: json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file)
key = (item.get('sha256'), item.get('filename'), item.get('hostname')) json_output['response']['exechistories'].clear()
seen[key] = item date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()
deduplicated = list(seen.values()) percentage_diff = (((days + 10) - date_diff.days) / (days + 10)) * 100
with open(file_path, 'w') as file: pbar.n = round(percentage_diff)
json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) pbar.set_description_str(f"Total of {policiesnames} Complete: ")
json_output['response']['exechistories'].clear() pbar.refresh()
filebar.n = 1
with open(file_path, 'r') as file: with open(file_path, 'r') as file:
final_output = json.load(file) final_output = json.load(file)
os.remove(file_path) os.remove(file_path)
@@ -144,7 +147,7 @@ def skipback(days):
Generate a MongoDB ObjectId for a given number of days ago from today. Generate a MongoDB ObjectId for a given number of days ago from today.
Adds 1 extra day to the input to look further back. Adds 1 extra day to the input to look further back.
""" """
adjusted_days = days + 1 adjusted_days = days + 10
date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days) date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days)
timestamp = int(date_days_ago.timestamp()) timestamp = int(date_days_ago.timestamp())
hex_timestamp = format(timestamp, '08x') hex_timestamp = format(timestamp, '08x')