diff --git a/AirlockTools.py b/AirlockTools.py index 182e0f7..1977229 100644 --- a/AirlockTools.py +++ b/AirlockTools.py @@ -291,7 +291,7 @@ def menu_prepare_to_enforce(): executionhist_policy1 = executionhist_policy1.sort_values(by=['sha256','filename']) executionhist_policy1.to_csv(f"dataframe_csv\\executionhist_{first_policy}.csv", index=False) - ct.style_dataframe_dark(executionhist_policy1, f"dataframe_html\\executionhist_{first_policy}.html") + #ct.style_dataframe_dark(executionhist_policy1, f"dataframe_html\\executionhist_{first_policy}.html") print(ct.colorText(f"Staging of Execution history for policy: {first_policy} is complete","green")) if not os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"): @@ -303,7 +303,7 @@ def menu_prepare_to_enforce(): executionhist_policy2 = executionhist_policy2.sort_values(by=['sha256','filename']) executionhist_policy2.to_csv(f"dataframe_csv\\executionhist_{second_policy}.csv", index=False) - ct.style_dataframe_dark(executionhist_policy2, f"dataframe_html\\executionhist_{second_policy}.html") + #ct.style_dataframe_dark(executionhist_policy2, f"dataframe_html\\executionhist_{second_policy}.html") print(ct.colorText(f"Staging of Exection history for policy: {first_policy} is complete","green")) #Combine the two policies execution histories @@ -311,20 +311,20 @@ def menu_prepare_to_enforce(): if second_policy is first_policy: execuctionhist_combined = executionhist_policy1 execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False) - ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html") + #ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html") print(ct.colorText(f"Dataframes have been combined","green")) elif os.path.exists(f"dataframe_csv\\executionhist_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"): execuctionhist_combined = pd.concat([tryToReadCSV(f"dataframe_csv\\executionhist_{first_policy}.csv") , tryToReadCSV(f"dataframe_csv\\executionhist_{second_policy}.csv")], ignore_index=True).sort_values(by=['sha256','filename']) execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False) - ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html") + #ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html") print(ct.colorText(f"Dataframes have been combined","green")) #Keep only unique combinations of hash, filename, and hostname if f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv": unique_executions = tryToReadCSV(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv").drop_duplicates(subset=['sha256', 'filename', 'hostname']) unique_executions.to_csv(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv") - ct.style_dataframe_dark(unique_executions, f"dataframe_html\\unique_execuctions.html") + #ct.style_dataframe_dark(unique_executions, f"dataframe_html\\unique_execuctions.html") #Add Hash info to the combined execution history @@ -332,7 +332,7 @@ def menu_prepare_to_enforce(): print(ct.colorText(f"Preparing to pull hash info","green")) augmented_combo= utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv")) augmented_combo.to_csv(f"dataframe_csv\\augmented_combo_{first_policy}_{second_policy}.csv", index=False) - ct.style_dataframe_dark(augmented_combo, f"dataframe_html\\augmented_combo_{first_policy}_{second_policy}.html") + #ct.style_dataframe_dark(augmented_combo, f"dataframe_html\\augmented_combo_{first_policy}_{second_policy}.html") print(ct.colorText(f"Hash reputation info added to dataframe","green")) #Categorize the hashes @@ -345,19 +345,19 @@ def menu_prepare_to_enforce(): categorized[0].sort_values(by=['sha256','filename_x']) categorized[0].to_csv(f"dataframe_csv\\hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False) - ct.style_dataframe_dark(categorized[0], f"dataframe_html\\hashes_needing_approval_{first_policy}_{second_policy}.html") + #ct.style_dataframe_dark(categorized[0], f"dataframe_html\\hashes_needing_approval_{first_policy}_{second_policy}.html") if not categorized[1].empty: categorized[1].sort_values(by=['sha256','filename_x']) categorized[1].to_csv(f"dataframe_csv\\automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False) - ct.style_dataframe_dark(categorized[1], f"dataframe_html\\automatically_approved_hashes_{first_policy}_{second_policy}.html") + #ct.style_dataframe_dark(categorized[1], f"dataframe_html\\automatically_approved_hashes_{first_policy}_{second_policy}.html") if not categorized[2].empty: categorized[2].sort_values(by=['sha256','filename_x']) categorized[2].to_csv(f"dataframe_csv\\unapproved_hashes__{first_policy}_{second_policy}.csv", index=False) - ct.style_dataframe_dark(categorized[2], f"dataframe_html\\unapproved_hashes_{first_policy}_{second_policy}.html") + #ct.style_dataframe_dark(categorized[2], f"dataframe_html\\unapproved_hashes_{first_policy}_{second_policy}.html") print(ct.colorText(f"Hashes have been categorized","green")) diff --git a/utils/allowlist.py b/utils/allowlist.py index 894bbae..d533d79 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -22,9 +22,7 @@ import utils.pretty as ct import ijson def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): - if not os.path.exists('chunkinator.csv'): - with open('chunkinator.csv', 'x') as file: - pass + file_path = 'chunkinator.json' headers = { "X-APIKey": os.getenv('APIKEY') } @@ -39,27 +37,48 @@ def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool): if array_dividend == 0: array_dividend == 1 for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]): - if (datetime.date.today() - datetime.timedelta(days=30) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): - print("Found Date Match") + if (datetime.date.today() - datetime.timedelta(days=60) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): match_found = True break checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend) - print(ct.colorText(f"{checkpoints_processed} checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) + if (index + 1) < checkpoints_processed: + print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed and Found with Date Match. Last Checkpoint: {item['checkpoint']}.", "blue")) + else: + print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed. Last Checkpoint: {item['checkpoint']}.", "blue")) checkpoint = item['checkpoint'] if match_found == True: for index, item in enumerate(json_response_data['response']['exechistories']): if index == len(json_response_data['response']['exechistories']) -1: checkpoint = item['checkpoint'] - print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue")) + print(ct.colorText(f"All Events Processed. Next Checkpoint: {item['checkpoint']}", "blue")) break else: - if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): + if (datetime.date.today() - datetime.timedelta(days=60) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()): pass else: - print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}") + #print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}") json_output['response']['exechistories'].append(item) - match_found = False + if os.path.exists(file_path): + seen = {} + with open(file_path, 'r') as file: + temp_file = json.load(file) + combined_data = temp_file['response']['exechistories'] + json_output['response']['exechistories'] + for item in combined_data: + key = (item.get('sha256'), item.get('filename'), item.get('hostname')) + seen[key] = item + deduplicated_data = list(seen.values()) + with open(file_path, 'w') as file: + json.dump({'error': 'Success', 'response': {'exechistories': deduplicated_data}}, file) + json_output = {'error': 'Success', 'response': {'exechistories': []}} + else: + with open(file_path, 'a') as file: + json.dump(json_output, file) + json_output = {'error': 'Success', 'response': {'exechistories': []}} + + with open(file_path, 'r') as file: + json_output = json.load(file) json_output = json.dumps(json_output) + os.remove(file_path) if outputjson == True: return json_output