AirlockTools.py

@@ -433,9 +433,22 @@ def menu_prepare_to_enforce():
 
 					allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False)
 
+<<<<<<< HEAD
 					allowbyhash.sort_values(by=["filename"])
 					
 					ct.style_dataframe_dark(allowbyhash, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
+=======
+					easyview = allowbyhash.groupby('sha256').agg(list).reset_index()
+					# Deduplicate and sort by first item in list
+					for col in easyview.columns:
+						if col != 'sha256':
+							easyview[col] = easyview[col].apply(lambda x: list(set(x)))
+							if col in ["reputation_status", "filename"]:
+					        	# Sort the list to ensure consistent first item
+								easyview[col] = easyview[col].apply(lambda x: sorted(x)[0] if x else None)
+			
+					ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
+>>>>>>> 3f2717d972d811b0615f9b250aeb5191b255aa47
 					ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html")
 
 					

requirements.txt

@@ -2,3 +2,4 @@ pandas==2.3.2
 python-dotenv==1.1.1
 Requests==2.32.5
 urllib3==2.5.0
+tqdm

utils/allowlist.py

@@ -21,6 +21,8 @@ import ijson
 import os
 from bson import ObjectId
 import datetime
+import tqdm
+import sys
 
 def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
     file_path = 'chunkinator.json'
@@ -33,32 +35,27 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
     headers = {"X-APIKey": os.getenv('APIKEY')}
     checkpoint = str(skipback(days))
     json_output = {'error': 'Success', 'response': {'exechistories': []}}
+    with tqdm.tqdm(file=sys.stdout, leave=True, total=10000, desc=f"Checkpoint Progess: {checkpoint}", colour="blue", initial=1) as filebar:
+        with tqdm.tqdm(file=sys.stdout, leave=True, total=100, desc=f"Total of {policiesnames} Complete: ") as pbar:
                 while True:
                     json_response_data = checkpoint_stomper(checkpoint, url, policiesnames, headers)
                     histories = json_response_data['response']['exechistories']
+                    filebar.total=len(histories)
                     if not histories:
                         break
-        array_dividend =  max(round(len(histories) / 20), 1)
-        match_found = False
-        for index, item in enumerate(histories[::array_dividend]):
-            if (datetime.date.today() - datetime.timedelta(days=days) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
                     match_found = True
-                break
-        checkpoints_processed = round(len(histories) / array_dividend)
-        if ( index + 1 ) < checkpoints_processed:
-            print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed with date match. Last Checkpoint: {checkpoint}", "blue"))
-        else:
-            print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) Processed. Last Checkpoint: {checkpoint}", "blue"))
                     if match_found == True:
                         for index, item in enumerate(histories):
                             if index == len(histories) - 1:
-                    print(ct.colorText(f"All Events Processed for {checkpoint}", "blue"))
                                 checkpoint = item['checkpoint']
+                                filebar.desc = f"Checkpoint Progress: {checkpoint}"
                                 break
                             else:
                                 if (datetime.date.today() - datetime.timedelta(days=days) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
                                     pass
                                 else: json_output['response']['exechistories'].append(item)
+                            filebar.update(1)
+                            filebar.refresh()
                     seen = {}
                     if os.path.exists(file_path):
                         with open(file_path, 'r') as file:
@@ -73,6 +70,12 @@ def pullPolicyExechistories(url, policiesnames, days, outputjson: bool):
                     with open(file_path, 'w') as file:
                         json.dump({'error': 'Success', 'response': {'exechistories': deduplicated}}, file)
                     json_output['response']['exechistories'].clear()
+                    date_diff = datetime.date.today() - datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()
+                    percentage_diff = (((days + 10) - date_diff.days) / (days + 10)) * 100
+                    pbar.n = round(percentage_diff)
+                    pbar.set_description_str(f"Total of {policiesnames} Complete: ")
+                    pbar.refresh()
+                    filebar.n = 1
     with open(file_path, 'r') as file:
         final_output = json.load(file)
     os.remove(file_path)
@@ -144,7 +147,7 @@ def skipback(days):
     Generate a MongoDB ObjectId for a given number of days ago from today.
     Adds 1 extra day to the input to look further back.
     """
-    adjusted_days = days + 1
+    adjusted_days = days + 10
     date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days)
     timestamp = int(date_days_ago.timestamp())
     hex_timestamp = format(timestamp, '08x')