From 2f47ee7e44b2c97b9679a1fd7b92668e0609bb5a Mon Sep 17 00:00:00 2001 From: brotoskyj Date: Wed, 3 Sep 2025 11:31:59 -0400 Subject: [PATCH 1/4] Quality of Life updates --- requirements.txt | 1 + utils/allowlist.py | 78 ++++++++++++++++++++++++---------------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3ba3d18..015b684 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pandas==2.3.2 python-dotenv==1.1.1 Requests==2.32.5 urllib3==2.5.0 +tqdm \ No newline at end of file diff --git a/utils/allowlist.py b/utils/allowlist.py index 4f9af7f..2d0f621 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -21,6 +21,8 @@ import ijson import os from bson import ObjectId import datetime +import tqdm +import time def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): file_path = 'chunkinator.json' @@ -33,46 +35,46 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): headers = {"X-APIKey": os.getenv('APIKEY')} checkpoint = str(skipback(days)) json_output = {'error': 'Success', 'response': {'exechistories': []}} - while True: - json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) - histories = json_response_data['response']['exechistories'] - if not histories: - break - array_dividend = max(round(len(histories) / 20), 1) - match_found = False - for index, item in enumerate(histories[::array_dividend]): - if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - match_found = True + with tqdm.tqdm(total=100, desc="Total Percentage Complete: ") as pbar: + while True: + json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) + histories = json_response_data['response']['exechistories'] + if not histories: break - checkpoints_processed = round(len(histories) / array_dividend) - if ( index + 1 ) < checkpoints_processed: - print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue")) - else: - print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue")) - if match_found == True: - for index, item in enumerate(histories): - if index == len(histories) - 1: - print(ct.colorText(f"All Events Processed for {checkpoint}", "blue")) - checkpoint = item['checkpoint'] + array_dividend = max(round(len(histories) / 20), 1) + match_found = False + for index, item in enumerate(histories[::array_dividend]): + if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + match_found = True break - else: - if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - pass - else: json_output['response']['exechistories'].append(item) - seen = {} - if os.path.exists(file_path): - with open(file_path, 'r') as file: - existing_data = json.load(file) - combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] - else: - combined = json_output['response']['exechistories'] - for item in combined: - key = (item.get('sha256'), item.get('filename'), item.get('hostname')) - seen[key] = item - deduplicated = list(seen.values()) - with open(file_path, 'w') as file: - json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) - json_output['response']['exechistories'].clear() + if match_found == True: + for index, item in enumerate(tqdm.tqdm(histories, desc=f"Processing events for {checkpoint}", unit="events", colour="blue", initial=1)): + if index == len(histories) - 1: + checkpoint = item['checkpoint'] + break + else: + if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + pass + else: json_output['response']['exechistories'].append(item) + seen = {} + if os.path.exists(file_path): + with open(file_path, 'r') as file: + existing_data = json.load(file) + combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] + else: + combined = json_output['response']['exechistories'] + for item in combined: + key = (item.get('sha256'), item.get('filename'), item.get('hostname')) + seen[key] = item + deduplicated = list(seen.values()) + with open(file_path, 'w') as file: + json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) + json_output['response']['exechistories'].clear() + date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date() + percentage_diff = ((days - date_diff.days) / 60) * 100 + pbar.n = round(percentage_diff) + pbar.set_description_str(f"Total Percentage Complete: ({percentage_diff:.1f}%)") + pbar.refresh with open(file_path, 'r') as file: final_output = json.load(file) os.remove(file_path) From c428a4a989173438c51404190ecb44eb35bde9c6 Mon Sep 17 00:00:00 2001 From: brotoskyj Date: Wed, 3 Sep 2025 11:53:59 -0400 Subject: [PATCH 2/4] Minor Changes to Quality of Life Update --- utils/allowlist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/allowlist.py b/utils/allowlist.py index 2d0f621..b087877 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -35,7 +35,7 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): headers = {"X-APIKey": os.getenv('APIKEY')} checkpoint = str(skipback(days)) json_output = {'error': 'Success', 'response': {'exechistories': []}} - with tqdm.tqdm(total=100, desc="Total Percentage Complete: ") as pbar: + with tqdm.tqdm(total=100, desc=f"Total of {policiesnames} Complete: ") as pbar: while True: json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) histories = json_response_data['response']['exechistories'] @@ -73,7 +73,7 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date() percentage_diff = ((days - date_diff.days) / 60) * 100 pbar.n = round(percentage_diff) - pbar.set_description_str(f"Total Percentage Complete: ({percentage_diff:.1f}%)") + pbar.set_description_str(f"Total of {policiesnames} Complete: ({percentage_diff:.1f}%)") pbar.refresh with open(file_path, 'r') as file: final_output = json.load(file) From 40def7307575568aac23b1813e2e63bf7da9cb40 Mon Sep 17 00:00:00 2001 From: brotoskyj Date: Wed, 3 Sep 2025 16:42:56 -0400 Subject: [PATCH 3/4] Minor Changes to Quality of Life Update --- utils/allowlist.py | 81 +++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/utils/allowlist.py b/utils/allowlist.py index b087877..0f02958 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -22,7 +22,7 @@ import os from bson import ObjectId import datetime import tqdm -import time +import sys def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): file_path = 'chunkinator.json' @@ -35,46 +35,47 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): headers = {"X-APIKey": os.getenv('APIKEY')} checkpoint = str(skipback(days)) json_output = {'error': 'Success', 'response': {'exechistories': []}} - with tqdm.tqdm(total=100, desc=f"Total of {policiesnames} Complete: ") as pbar: - while True: - json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) - histories = json_response_data['response']['exechistories'] - if not histories: - break - array_dividend = max(round(len(histories) / 20), 1) - match_found = False - for index, item in enumerate(histories[::array_dividend]): - if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - match_found = True - break - if match_found == True: - for index, item in enumerate(tqdm.tqdm(histories, desc=f"Processing events for {checkpoint}", unit="events", colour="blue", initial=1)): - if index == len(histories) - 1: - checkpoint = item['checkpoint'] + with tqdm.tqdm(file=sys.stdout, leave=True, total=10000, desc=f"Checkpoint Progess: {checkpoint}", colour="blue", initial=1) as filebar: + with tqdm.tqdm(file=sys.stdout, leave=True, total=100, desc=f"Total of {policiesnames} Complete: ") as pbar: + while True: + json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) + histories = json_response_data['response']['exechistories'] + filebar.total=len(histories) + if not histories: break + match_found = True + if match_found == True: + for index, item in enumerate(histories): + if index == len(histories) - 1: + checkpoint = item['checkpoint'] + filebar.desc = f"Checkpoint Progress: {checkpoint}" + break + else: + if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + pass + else: json_output['response']['exechistories'].append(item) + filebar.update(1) + filebar.refresh() + seen = {} + if os.path.exists(file_path): + with open(file_path, 'r') as file: + existing_data = json.load(file) + combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] else: - if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - pass - else: json_output['response']['exechistories'].append(item) - seen = {} - if os.path.exists(file_path): - with open(file_path, 'r') as file: - existing_data = json.load(file) - combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] - else: - combined = json_output['response']['exechistories'] - for item in combined: - key = (item.get('sha256'), item.get('filename'), item.get('hostname')) - seen[key] = item - deduplicated = list(seen.values()) - with open(file_path, 'w') as file: - json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) - json_output['response']['exechistories'].clear() - date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date() - percentage_diff = ((days - date_diff.days) / 60) * 100 - pbar.n = round(percentage_diff) - pbar.set_description_str(f"Total of {policiesnames} Complete: ({percentage_diff:.1f}%)") - pbar.refresh + combined = json_output['response']['exechistories'] + for item in combined: + key = (item.get('sha256'), item.get('filename'), item.get('hostname')) + seen[key] = item + deduplicated = list(seen.values()) + with open(file_path, 'w') as file: + json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) + json_output['response']['exechistories'].clear() + date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date() + percentage_diff = (((days + 10) - date_diff.days) / (days + 10)) * 100 + pbar.n = round(percentage_diff) + pbar.set_description_str(f"Total of {policiesnames} Complete: ") + pbar.refresh() + filebar.n = 1 with open(file_path, 'r') as file: final_output = json.load(file) os.remove(file_path) @@ -146,7 +147,7 @@ def skipback(days): Generate a MongoDB ObjectId for a given number of days ago from today. Adds 1 extra day to the input to look further back. """ - adjusted_days = days + 1 + adjusted_days = days + 10 date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days) timestamp = int(date_days_ago.timestamp()) hex_timestamp = format(timestamp, '08x') From 3f2717d972d811b0615f9b250aeb5191b255aa47 Mon Sep 17 00:00:00 2001 From: brotoskyj Date: Wed, 3 Sep 2025 17:04:47 -0400 Subject: [PATCH 4/4] Fixed List --- AirlockTools.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/AirlockTools.py b/AirlockTools.py index e8dc74e..0e13021 100644 --- a/AirlockTools.py +++ b/AirlockTools.py @@ -434,12 +434,13 @@ def menu_prepare_to_enforce(): allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False) easyview = allowbyhash.groupby('sha256').agg(list).reset_index() - # Deduplicate all list columns in easyview + # Deduplicate and sort by first item in list for col in easyview.columns: - if col != 'sha256': # Skip the grouping column + if col != 'sha256': easyview[col] = easyview[col].apply(lambda x: list(set(x))) - - easyview = easyview.sort_values(by=["reputation_status", "filename"]) + if col in ["reputation_status", "filename"]: + # Sort the list to ensure consistent first item + easyview[col] = easyview[col].apply(lambda x: sorted(x)[0] if x else None) ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html") ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html")