From f081594a7edf71a43762a72f6e28cb88bcae8fd4 Mon Sep 17 00:00:00 2001 From: brotoskyj Date: Mon, 25 Aug 2025 23:36:24 -0400 Subject: [PATCH] closes #15 Created iterator to skip over items. This was causing issues because it was too many entries and it would cause a memory leak. --- utils/allowlist.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/utils/allowlist.py b/utils/allowlist.py index f9c6d7a..46c48aa 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -20,7 +20,6 @@ import time import utils.colortext as ct import ijson def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): - headers = { "X-APIKey": os.getenv('APIKEY') } @@ -30,17 +29,31 @@ def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): json_response_data = checkpoint_stomper(checkpoint, url, policiesnames[choice], headers) if not json_response_data['response']['exechistories']: break - for index, item in enumerate(json_response_data['response']['exechistories']): - if index == len(json_response_data['response']['exechistories']) -1: - checkpoint = item['checkpoint'] - print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) + match_found = False + array_dividend = round(len(json_response_data['response']['exechistories'])/20) + if array_dividend == 0: + array_dividend == 1 + for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]): + if (datetime.date.today() - datetime.timedelta(days=30) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + print("Found Date Match") + match_found = True break - else: - if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - print(item['datetime']) + checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend) + print(ct.colorText(f"{checkpoints_processed} checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) + checkpoint = item['checkpoint'] + if match_found == True: + for index, item in enumerate(json_response_data['response']['exechistories']): + if index == len(json_response_data['response']['exechistories']) -1: + checkpoint = item['checkpoint'] + print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) + break else: - print(item['datetime']) - json_output['response']['exechistories'].append(item) + if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + pass + else: + print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}") + json_output['response']['exechistories'].append(item) + match_found = False json_output = json.dumps(json_output) if outputjson == True: return json_output @@ -57,7 +70,7 @@ def checkpoint_stomper(checkpoint, url, policy, headers): with requests.request("POST", endpoint, headers=headers, data=payload, verify=False, stream=True) as response: parser = ijson.items(response.raw, 'response.exechistories.item') for item in parser: - key = (item.get('sha256'), item.get('hostname')), item.get('datetime') + key = (item.get('sha256'), item.get('hostname')) if key not in json_output: json_output['response']['exechistories'].append(item) parse_text = json.loads(json.dumps(json_output))