diff --git a/airlock_libs/Cargo.lock b/airlock_libs/Cargo.lock index eaaadaa..e14b90e 100644 --- a/airlock_libs/Cargo.lock +++ b/airlock_libs/Cargo.lock @@ -26,7 +26,7 @@ dependencies = [ [[package]] name = "airlock_libs" -version = "7.1.0" +version = "7.3.0" dependencies = [ "chrono", "crossbeam", diff --git a/airlock_libs/Cargo.toml b/airlock_libs/Cargo.toml index 4833a01..8827af5 100644 --- a/airlock_libs/Cargo.toml +++ b/airlock_libs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "airlock_libs" -version = "7.1.0" +version = "7.3.0" edition = "2024" [dependencies] diff --git a/airlock_libs/pyproject.toml b/airlock_libs/pyproject.toml index aee7519..5060fe8 100644 --- a/airlock_libs/pyproject.toml +++ b/airlock_libs/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "airlock_libs" -version = "7.1.0" +version = "7.3.0" description = "Airlock Digital API Wrapper" readme = "README.md" license = { text = "AGPL-3.0-only" } diff --git a/airlock_libs/src/services.rs b/airlock_libs/src/services.rs index 767598d..3023f45 100644 --- a/airlock_libs/src/services.rs +++ b/airlock_libs/src/services.rs @@ -106,7 +106,7 @@ pub fn pull_policy_exec_histories( }); let cutoff: chrono::NaiveDateTime = Local::now().naive_local() - chrono::Duration::days(days); - let (tx, rx) = unbounded::>(); + let (tx, rx) = unbounded::<(Context, Vec)>(); let pb_clone = progress_bar.clone(); thread::spawn(move || { let mut seen: HashMap<(String, String, String), Group> = if writeable_filepath.exists() @@ -137,7 +137,14 @@ pub fn pull_policy_exec_histories( } else { HashMap::new() }; - while let Ok(parsed_responses) = rx.recv() { + while let Ok((cx, parsed_responses)) = rx.recv() { + cx.span().add_event( + "Received Data from Producer", + vec![KeyValue::new( + "Items to Process", + parsed_responses.len().to_string(), + )], + ); for executions in parsed_responses { if executions.checkpoint.is_empty() || executions.datetime.is_empty() { continue; @@ -166,11 +173,28 @@ pub fn pull_policy_exec_histories( }; let data_write: String = serde_json::to_string_pretty(&final_response).unwrap(); match fs::write(&writeable_filepath, data_write) { - Ok(_) => {} + Ok(_) => { + cx.span().add_event( + "Writing Data to File", + vec![KeyValue::new("Success", "Ok".to_string())], + ); + } Err(e) => { - println!("Failed to write to: {:?}: {}", &writeable_filepath, e); + cx.span().add_event( + "Writing Data to File", + vec![KeyValue::new("Failed", e.to_string())], + ); + cx.span() + .set_status(Status::error("Failed to Write to File")); } } + cx.span().add_event( + "Finished Deduplicating Data", + vec![KeyValue::new( + "Items Successfully Processed", + seen.len().to_string(), + )], + ); } }); let mut first_date: Option = None; @@ -193,13 +217,28 @@ pub fn pull_policy_exec_histories( )); loop { let execution_histories = tracer.in_span(checkpoint_number.to_string(), |cx| { - let results: ApiResponse = history_logging( + cx.span().add_event( + "Retrieving Responses from API", + vec![KeyValue::new( + "Checkpoint Number", + checkpoint_number.to_string(), + )], + ); + let results: ApiResponse = rt.block_on(history_logging( &base_url, &exec_types, &checkpoint_number, &policy_names, &client, + )); + cx.span().add_event( + "Got Responses from API", + vec![KeyValue::new( + "Items in Response", + results.response.exechistories.len().to_string(), + )], ); + cx.span().set_status(Status::Ok); cx.span().set_attribute(KeyValue::new( "items_in_response", results.response.exechistories.len().to_string(), @@ -210,7 +249,16 @@ pub fn pull_policy_exec_histories( if parsed_responses.is_empty() { break; } - tx.send(parsed_responses.clone()).unwrap(); + match tx.send((cx.clone(), parsed_responses.clone())) { + Ok(_) => {} + Err(e) => { + cx.span().add_event( + "Failed to Send Items to Processor", + vec![KeyValue::new("Response from Processor", e.to_string())], + ); + cx.span().set_status(Status::error("Processor Failed")) + } + } checkpoint_number = parsed_responses.last().unwrap().checkpoint.clone(); if let Some(last_item) = parsed_responses.last() && let Ok(last_date) = NaiveDate::parse_from_str( @@ -263,7 +311,7 @@ fn build_client(headers: HeaderMap) -> Result { .build() } -#[tokio::main] +#[tracing::instrument(name = "history_logging")] async fn history_logging( base_url: &String, exec_types: &String, @@ -292,7 +340,7 @@ async fn history_logging( Ok(res) => { let first_response: ApiResponse = serde_json::from_str(&res.text().await.unwrap()) .expect("Failed to retrieve response from API"); - return first_response; + first_response } Err(_res) => { let failed_response: ApiResponse = ApiResponse { @@ -301,7 +349,7 @@ async fn history_logging( exechistories: vec![], }, }; - return failed_response; + failed_response } } } diff --git a/requirements.txt b/requirements.txt index 6862175..d3c4959 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,4 @@ pyperclip==1.11.0 # Custom/Private packages --extra-index-url https://git.racooncity.org/api/packages/brotoskyj/pypi/simple/ -airlock_libs==7.1.0 \ No newline at end of file +airlock_libs==7.3.0 \ No newline at end of file