diff --git a/AirlockTools.py b/AirlockTools.py index 6e7955d..3fd773b 100644 --- a/AirlockTools.py +++ b/AirlockTools.py @@ -433,9 +433,22 @@ def menu_prepare_to_enforce(): allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False) +<<<<<<< HEAD allowbyhash.sort_values(by=["filename"]) ct.style_dataframe_dark(allowbyhash, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html") +======= + easyview = allowbyhash.groupby('sha256').agg(list).reset_index() + # Deduplicate and sort by first item in list + for col in easyview.columns: + if col != 'sha256': + easyview[col] = easyview[col].apply(lambda x: list(set(x))) + if col in ["reputation_status", "filename"]: + # Sort the list to ensure consistent first item + easyview[col] = easyview[col].apply(lambda x: sorted(x)[0] if x else None) + + ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html") +>>>>>>> 3f2717d972d811b0615f9b250aeb5191b255aa47 ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html") diff --git a/requirements.txt b/requirements.txt index 3ba3d18..015b684 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pandas==2.3.2 python-dotenv==1.1.1 Requests==2.32.5 urllib3==2.5.0 +tqdm \ No newline at end of file diff --git a/utils/allowlist.py b/utils/allowlist.py index 4f9af7f..0f02958 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -21,6 +21,8 @@ import ijson import os from bson import ObjectId import datetime +import tqdm +import sys def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): file_path = 'chunkinator.json' @@ -33,46 +35,47 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): headers = {"X-APIKey": os.getenv('APIKEY')} checkpoint = str(skipback(days)) json_output = {'error': 'Success', 'response': {'exechistories': []}} - while True: - json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) - histories = json_response_data['response']['exechistories'] - if not histories: - break - array_dividend = max(round(len(histories) / 20), 1) - match_found = False - for index, item in enumerate(histories[::array_dividend]): - if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - match_found = True - break - checkpoints_processed = round(len(histories) / array_dividend) - if ( index + 1 ) < checkpoints_processed: - print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue")) - else: - print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue")) - if match_found == True: - for index, item in enumerate(histories): - if index == len(histories) - 1: - print(ct.colorText(f"All Events Processed for {checkpoint}", "blue")) - checkpoint = item['checkpoint'] - break - else: - if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - pass - else: json_output['response']['exechistories'].append(item) - seen = {} - if os.path.exists(file_path): - with open(file_path, 'r') as file: - existing_data = json.load(file) - combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] - else: - combined = json_output['response']['exechistories'] - for item in combined: - key = (item.get('sha256'), item.get('filename'), item.get('hostname')) - seen[key] = item - deduplicated = list(seen.values()) - with open(file_path, 'w') as file: - json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) - json_output['response']['exechistories'].clear() + with tqdm.tqdm(file=sys.stdout, leave=True, total=10000, desc=f"Checkpoint Progess: {checkpoint}", colour="blue", initial=1) as filebar: + with tqdm.tqdm(file=sys.stdout, leave=True, total=100, desc=f"Total of {policiesnames} Complete: ") as pbar: + while True: + json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) + histories = json_response_data['response']['exechistories'] + filebar.total=len(histories) + if not histories: + break + match_found = True + if match_found == True: + for index, item in enumerate(histories): + if index == len(histories) - 1: + checkpoint = item['checkpoint'] + filebar.desc = f"Checkpoint Progress: {checkpoint}" + break + else: + if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + pass + else: json_output['response']['exechistories'].append(item) + filebar.update(1) + filebar.refresh() + seen = {} + if os.path.exists(file_path): + with open(file_path, 'r') as file: + existing_data = json.load(file) + combined = existing_data['response']['exechistories'] + json_output['response']['exechistories'] + else: + combined = json_output['response']['exechistories'] + for item in combined: + key = (item.get('sha256'), item.get('filename'), item.get('hostname')) + seen[key] = item + deduplicated = list(seen.values()) + with open(file_path, 'w') as file: + json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) + json_output['response']['exechistories'].clear() + date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date() + percentage_diff = (((days + 10) - date_diff.days) / (days + 10)) * 100 + pbar.n = round(percentage_diff) + pbar.set_description_str(f"Total of {policiesnames} Complete: ") + pbar.refresh() + filebar.n = 1 with open(file_path, 'r') as file: final_output = json.load(file) os.remove(file_path) @@ -144,7 +147,7 @@ def skipback(days): Generate a MongoDB ObjectId for a given number of days ago from today. Adds 1 extra day to the input to look further back. """ - adjusted_days = days + 1 + adjusted_days = days + 10 date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days) timestamp = int(date_days_ago.timestamp()) hex_timestamp = format(timestamp, '08x')