Working Chunkinantor
This commit is contained in:
brotoskyj
+9
-9
@@ -291,7 +291,7 @@ def menu_prepare_to_enforce():
|
||||
executionhist_policy1 = executionhist_policy1.sort_values(by=['sha256','filename'])
|
||||
|
||||
executionhist_policy1.to_csv(f"dataframe_csv\\executionhist_{first_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(executionhist_policy1, f"dataframe_html\\executionhist_{first_policy}.html")
|
||||
#ct.style_dataframe_dark(executionhist_policy1, f"dataframe_html\\executionhist_{first_policy}.html")
|
||||
print(ct.colorText(f"Staging of Execution history for policy: {first_policy} is complete","green"))
|
||||
|
||||
if not os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"):
|
||||
@@ -303,7 +303,7 @@ def menu_prepare_to_enforce():
|
||||
executionhist_policy2 = executionhist_policy2.sort_values(by=['sha256','filename'])
|
||||
|
||||
executionhist_policy2.to_csv(f"dataframe_csv\\executionhist_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(executionhist_policy2, f"dataframe_html\\executionhist_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(executionhist_policy2, f"dataframe_html\\executionhist_{second_policy}.html")
|
||||
print(ct.colorText(f"Staging of Exection history for policy: {first_policy} is complete","green"))
|
||||
|
||||
#Combine the two policies execution histories
|
||||
@@ -311,20 +311,20 @@ def menu_prepare_to_enforce():
|
||||
if second_policy is first_policy:
|
||||
execuctionhist_combined = executionhist_policy1
|
||||
execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html")
|
||||
print(ct.colorText(f"Dataframes have been combined","green"))
|
||||
|
||||
elif os.path.exists(f"dataframe_csv\\executionhist_{first_policy}.csv") and os.path.exists(f"dataframe_csv\\executionhist_{second_policy}.csv"):
|
||||
execuctionhist_combined = pd.concat([tryToReadCSV(f"dataframe_csv\\executionhist_{first_policy}.csv") , tryToReadCSV(f"dataframe_csv\\executionhist_{second_policy}.csv")], ignore_index=True).sort_values(by=['sha256','filename'])
|
||||
execuctionhist_combined.to_csv(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(df_aggregated_combo, f"dataframe_html\\execuctionhist_combined_{first_policy}_{second_policy}.html")
|
||||
print(ct.colorText(f"Dataframes have been combined","green"))
|
||||
|
||||
#Keep only unique combinations of hash, filename, and hostname
|
||||
if f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv":
|
||||
unique_executions = tryToReadCSV(f"dataframe_csv\\execuctionhist_combined_{first_policy}_{second_policy}.csv").drop_duplicates(subset=['sha256', 'filename', 'hostname'])
|
||||
unique_executions.to_csv(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv")
|
||||
ct.style_dataframe_dark(unique_executions, f"dataframe_html\\unique_execuctions.html")
|
||||
#ct.style_dataframe_dark(unique_executions, f"dataframe_html\\unique_execuctions.html")
|
||||
|
||||
|
||||
#Add Hash info to the combined execution history
|
||||
@@ -332,7 +332,7 @@ def menu_prepare_to_enforce():
|
||||
print(ct.colorText(f"Preparing to pull hash info","green"))
|
||||
augmented_combo= utils.hashfunctions.augmentAggregatedHashes(url,tryToReadCSV(f"dataframe_csv\\unique_executions{first_policy}_{second_policy}.csv"))
|
||||
augmented_combo.to_csv(f"dataframe_csv\\augmented_combo_{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(augmented_combo, f"dataframe_html\\augmented_combo_{first_policy}_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(augmented_combo, f"dataframe_html\\augmented_combo_{first_policy}_{second_policy}.html")
|
||||
print(ct.colorText(f"Hash reputation info added to dataframe","green"))
|
||||
|
||||
#Categorize the hashes
|
||||
@@ -345,19 +345,19 @@ def menu_prepare_to_enforce():
|
||||
categorized[0].sort_values(by=['sha256','filename_x'])
|
||||
|
||||
categorized[0].to_csv(f"dataframe_csv\\hashes_needing_approval_{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(categorized[0], f"dataframe_html\\hashes_needing_approval_{first_policy}_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(categorized[0], f"dataframe_html\\hashes_needing_approval_{first_policy}_{second_policy}.html")
|
||||
|
||||
if not categorized[1].empty:
|
||||
categorized[1].sort_values(by=['sha256','filename_x'])
|
||||
|
||||
categorized[1].to_csv(f"dataframe_csv\\automatically_approved_hashes_{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(categorized[1], f"dataframe_html\\automatically_approved_hashes_{first_policy}_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(categorized[1], f"dataframe_html\\automatically_approved_hashes_{first_policy}_{second_policy}.html")
|
||||
|
||||
if not categorized[2].empty:
|
||||
categorized[2].sort_values(by=['sha256','filename_x'])
|
||||
|
||||
categorized[2].to_csv(f"dataframe_csv\\unapproved_hashes__{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(categorized[2], f"dataframe_html\\unapproved_hashes_{first_policy}_{second_policy}.html")
|
||||
#ct.style_dataframe_dark(categorized[2], f"dataframe_html\\unapproved_hashes_{first_policy}_{second_policy}.html")
|
||||
|
||||
print(ct.colorText(f"Hashes have been categorized","green"))
|
||||
|
||||
|
||||
+29
-10
@@ -22,9 +22,7 @@ import utils.pretty as ct
|
||||
import ijson
|
||||
|
||||
def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool):
|
||||
if not os.path.exists('chunkinator.csv'):
|
||||
with open('chunkinator.csv', 'x') as file:
|
||||
pass
|
||||
file_path = 'chunkinator.json'
|
||||
headers = {
|
||||
"X-APIKey": os.getenv('APIKEY')
|
||||
}
|
||||
@@ -39,27 +37,48 @@ def pullPolicyExechistories(url, choice, policiesnames, outputjson: bool):
|
||||
if array_dividend == 0:
|
||||
array_dividend == 1
|
||||
for index, item in enumerate(json_response_data['response']['exechistories'][::array_dividend]):
|
||||
if (datetime.date.today() - datetime.timedelta(days=30) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
|
||||
print("Found Date Match")
|
||||
if (datetime.date.today() - datetime.timedelta(days=60) <= datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
|
||||
match_found = True
|
||||
break
|
||||
checkpoints_processed = round(len(json_response_data['response']['exechistories'])/array_dividend)
|
||||
print(ct.colorText(f"{checkpoints_processed} checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue"))
|
||||
if (index + 1) < checkpoints_processed:
|
||||
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed and Found with Date Match. Last Checkpoint: {item['checkpoint']}.", "blue"))
|
||||
else:
|
||||
print(ct.colorText(f"{index + 1}/{checkpoints_processed} checkpoint(s) from this execution have been processed. Last Checkpoint: {item['checkpoint']}.", "blue"))
|
||||
checkpoint = item['checkpoint']
|
||||
if match_found == True:
|
||||
for index, item in enumerate(json_response_data['response']['exechistories']):
|
||||
if index == len(json_response_data['response']['exechistories']) -1:
|
||||
checkpoint = item['checkpoint']
|
||||
print(ct.colorText(f"All checkpoints from this execution have been processed. Stepping to the subsequent checkpoint. {item['checkpoint']}", "blue"))
|
||||
print(ct.colorText(f"All Events Processed. Next Checkpoint: {item['checkpoint']}", "blue"))
|
||||
break
|
||||
else:
|
||||
if (datetime.date.today() - datetime.timedelta(days=30) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
|
||||
if (datetime.date.today() - datetime.timedelta(days=60) > datetime.datetime.strptime(item['datetime'].replace(' +0000 UTC', ''), '%Y-%m-%dT%H:%M:%SZ').date()):
|
||||
pass
|
||||
else:
|
||||
print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}")
|
||||
#print(f"Added: {item['datetime']} | {item['checkpoint']} | {item['filename']} | {item['sha256']}")
|
||||
json_output['response']['exechistories'].append(item)
|
||||
match_found = False
|
||||
if os.path.exists(file_path):
|
||||
seen = {}
|
||||
with open(file_path, 'r') as file:
|
||||
temp_file = json.load(file)
|
||||
combined_data = temp_file['response']['exechistories'] + json_output['response']['exechistories']
|
||||
for item in combined_data:
|
||||
key = (item.get('sha256'), item.get('filename'), item.get('hostname'))
|
||||
seen[key] = item
|
||||
deduplicated_data = list(seen.values())
|
||||
with open(file_path, 'w') as file:
|
||||
json.dump({'error': 'Success', 'response': {'exechistories': deduplicated_data}}, file)
|
||||
json_output = {'error': 'Success', 'response': {'exechistories': []}}
|
||||
else:
|
||||
with open(file_path, 'a') as file:
|
||||
json.dump(json_output, file)
|
||||
json_output = {'error': 'Success', 'response': {'exechistories': []}}
|
||||
|
||||
with open(file_path, 'r') as file:
|
||||
json_output = json.load(file)
|
||||
json_output = json.dumps(json_output)
|
||||
os.remove(file_path)
|
||||
if outputjson == True:
|
||||
return json_output
|
||||
|
||||
|
||||
Reference in New Issue
Block a user