mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Quality of Life updates

Browse Source
This commit is contained in:
brotoskyj
2025-09-03 11:31:59 -04:00
parent 935467fbcb
commit 2f47ee7e44
2 changed files with 41 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
+1
View File
@@ -2,3 +2,4 @@ pandas==2.3.2
python-dotenv==1.1.1 python-dotenv==1.1.1
Requests==2.32.5 Requests==2.32.5
urllib3==2.5.0 urllib3==2.5.0
tqdm
utils/allowlist.py
+9 -7
View File
@@ -21,6 +21,8 @@ import ijson
import os import os
from bson import ObjectId from bson import ObjectId
import datetime import datetime
import tqdm
import time
def pullPolicyExechistories(url, policiesnames, days, outputjson: bool): def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
file_path = 'chunkinator.json' file_path = 'chunkinator.json'
@@ -33,6 +35,7 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
headers = {"X-APIKey": os.getenv('APIKEY')} headers = {"X-APIKey": os.getenv('APIKEY')}
checkpoint = str(skipback(days)) checkpoint = str(skipback(days))
json_output = {'error': 'Success', 'response': {'exechistories': []}} json_output = {'error': 'Success', 'response': {'exechistories': []}}
with tqdm.tqdm(total=100, desc="Total Percentage Complete: ") as pbar:
while True: while True:
json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers) json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers)
histories = json_response_data['response']['exechistories'] histories = json_response_data['response']['exechistories']
@@ -44,15 +47,9 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
match_found = True match_found = True
break break
checkpoints_processed = round(len(histories) / array_dividend)
if ( index + 1 ) < checkpoints_processed:
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue"))
else:
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue"))
if match_found == True: if match_found == True:
for index, item in enumerate(histories): for index, item in enumerate(tqdm.tqdm(histories, desc=f"Processing events for {checkpoint}", unit="events", colour="blue", initial=1)):
if index == len(histories) - 1: if index == len(histories) - 1:
print(ct.colorText(f"All Events Processed for {checkpoint}", "blue"))
checkpoint = item['checkpoint'] checkpoint = item['checkpoint']
break break
else: else:
@@ -73,6 +70,11 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
with open(file_path, 'w') as file: with open(file_path, 'w') as file:
json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file) json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file)
json_output['response']['exechistories'].clear() json_output['response']['exechistories'].clear()
date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()
percentage_diff = ((days - date_diff.days) / 60) * 100
pbar.n = round(percentage_diff)
pbar.set_description_str(f"Total Percentage Complete: ({percentage_diff:.1f}%)")
pbar.refresh
with open(file_path, 'r') as file: with open(file_path, 'r') as file:
final_output = json.load(file) final_output = json.load(file)
os.remove(file_path) os.remove(file_path)