utils/allowlist.py

@@ -22,7 +22,7 @@ import os
 from bson import ObjectId
 import datetime
 import tqdm
-import time
+import sys
 
 def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
     file_path = 'chunkinator.json'
@@ -35,27 +35,27 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
     headers = {"X-APIKey": os.getenv('APIKEY')}
     checkpoint = str(skipback(days))
     json_output = {'error': 'Success', 'response': {'exechistories': []}}
-    with tqdm.tqdm(total=100, desc=f"Total of {policiesnames} Complete: ") as pbar:
+    with tqdm.tqdm(file=sys.stdout, leave=True, total=10000, desc=f"Checkpoint Progess: {checkpoint}", colour="blue", initial=1) as filebar:
+        with tqdm.tqdm(file=sys.stdout, leave=True, total=100, desc=f"Total of {policiesnames} Complete: ") as pbar:
                 while True:
                     json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers)
                     histories = json_response_data['response']['exechistories']
+                    filebar.total=len(histories)
                     if not histories:
                         break
-            array_dividend =  max(round(len(histories) / 20), 1)
-            match_found = False
-            for index, item in enumerate(histories[::array_dividend]):
-                if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
                     match_found = True
-                    break
                     if match_found == True:
-                for index, item in enumerate(tqdm.tqdm(histories, desc=f"Processing events for {checkpoint}", unit="events", colour="blue", initial=1)):
+                        for index, item in enumerate(histories):
                             if index == len(histories) - 1:
                                 checkpoint = item['checkpoint']
+                                filebar.desc = f"Checkpoint Progress: {checkpoint}"
                                 break
                             else:
                                 if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
                                     pass
                                 else: json_output['response']['exechistories'].append(item)
+                            filebar.update(1)
+                            filebar.refresh()
                     seen = {}
                     if os.path.exists(file_path):
                         with open(file_path, 'r') as file:
@@ -71,10 +71,11 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
                         json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file)
                     json_output['response']['exechistories'].clear()
                     date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()
-            percentage_diff = ((days - date_diff.days) / 60) * 100
+                    percentage_diff = (((days + 10) - date_diff.days) / (days + 10)) * 100
                     pbar.n = round(percentage_diff)
-            pbar.set_description_str(f"Total of {policiesnames} Complete: ({percentage_diff:.1f}%)")
-            pbar.refresh
+                    pbar.set_description_str(f"Total of {policiesnames} Complete: ")
+                    pbar.refresh()
+                    filebar.n = 1
     with open(file_path, 'r') as file:
         final_output = json.load(file)
     os.remove(file_path)
@@ -146,7 +147,7 @@ def skipback(days):
     Generate a MongoDB ObjectId for a given number of days ago from today.
     Adds 1 extra day to the input to look further back.
     """
-    adjusted_days = days + 1
+    adjusted_days = days + 10
     date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days)
     timestamp = int(date_days_ago.timestamp())
     hex_timestamp = format(timestamp, '08x')