From 5bbea5017103f3fc117e7f13d21d49a07ee9d532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 10 Oct 2023 12:48:47 +0200 Subject: [PATCH 01/46] feat: implement HttpHead as new RADType --- data_structures/src/chain/mod.rs | 10 ++++-- data_structures/src/proto/mod.rs | 2 ++ data_structures/src/serialization_helpers.rs | 4 +-- rad/src/lib.rs | 36 +++++++++++++++++--- schemas/witnet/witnet.proto | 3 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 34910b8a5..757f6c46e 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -1645,11 +1645,14 @@ pub enum RADType { /// HTTP POST request #[serde(rename = "HTTP-POST")] HttpPost, + /// HTTP HEAD request + #[serde(rename = "HTTP-HEAD")] + HttpHead, } impl RADType { pub fn is_http(&self) -> bool { - matches!(self, RADType::HttpGet | RADType::HttpPost) + matches!(self, RADType::HttpGet | RADType::HttpPost | RADType::HttpHead) } } @@ -1701,7 +1704,7 @@ pub struct RADRetrieve { pub script: Vec, /// Body of a HTTP-POST request pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-POST or HTTP-HEAD request pub headers: Vec<(String, String)>, } @@ -1809,7 +1812,8 @@ impl RADRetrieve { &[Field::Kind, Field::Url, Field::Script], &[Field::Body, Field::Headers], ) - } + }, + RADType::HttpHead => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) } } diff --git a/data_structures/src/proto/mod.rs b/data_structures/src/proto/mod.rs index 4d681d8b9..c1bb007ca 100644 --- a/data_structures/src/proto/mod.rs +++ b/data_structures/src/proto/mod.rs @@ -51,6 +51,7 @@ impl ProtobufConvert for chain::RADType { chain::RADType::HttpGet => witnet::DataRequestOutput_RADRequest_RADType::HttpGet, chain::RADType::Rng => witnet::DataRequestOutput_RADRequest_RADType::Rng, chain::RADType::HttpPost => witnet::DataRequestOutput_RADRequest_RADType::HttpPost, + chain::RADType::HttpHead => witnet::DataRequestOutput_RADRequest_RADType::HttpHead, } } @@ -60,6 +61,7 @@ impl ProtobufConvert for chain::RADType { witnet::DataRequestOutput_RADRequest_RADType::HttpGet => chain::RADType::HttpGet, witnet::DataRequestOutput_RADRequest_RADType::Rng => chain::RADType::Rng, witnet::DataRequestOutput_RADRequest_RADType::HttpPost => chain::RADType::HttpPost, + witnet::DataRequestOutput_RADRequest_RADType::HttpHead => chain::RADType::HttpHead, }) } } diff --git a/data_structures/src/serialization_helpers.rs b/data_structures/src/serialization_helpers.rs index 7f7d7d4e6..a8d59ee69 100644 --- a/data_structures/src/serialization_helpers.rs +++ b/data_structures/src/serialization_helpers.rs @@ -360,7 +360,7 @@ struct RADRetrieveSerializationHelperJson { /// Body of a HTTP-POST request #[serde(default, skip_serializing_if = "Vec::is_empty")] pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request #[serde(default, skip_serializing_if = "Vec::is_empty")] pub headers: Vec<(String, String)>, } @@ -377,7 +377,7 @@ struct RADRetrieveSerializationHelperBincode { pub script: Vec, /// Body of a HTTP-POST request pub body: Vec, - /// Extra headers of a HTTP-GET or HTTP-POST request + /// Extra headers of a HTTP-GET, HTTP-HEAD or HTTP-POST request pub headers: Vec<(String, String)>, } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 64c826780..a3781df42 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -25,10 +25,11 @@ use crate::{ create_radon_script_from_filters_and_reducer, execute_radon_script, unpack_radon_script, RadonScriptExecutionSettings, }, - types::{array::RadonArray, bytes::RadonBytes, string::RadonString, RadonTypes}, + types::{array::RadonArray, bytes::RadonBytes, map::RadonMap, string::RadonString, RadonTypes}, user_agents::UserAgent, }; use core::convert::From; +use std::collections::BTreeMap; use witnet_net::client::http::{WitnetHttpBody, WitnetHttpRequest}; pub mod conditions; @@ -173,6 +174,25 @@ fn string_response_with_data_report( execute_radon_script(input, &radon_script, context, settings) } +/// Handle HTTP-HEAD response with data, and return a `RadonReport`. +fn headers_response_with_data_report( + retrieve: &RADRetrieve, + response: &str, + context: &mut ReportContext, + settings: RadonScriptExecutionSettings, +) -> Result> { + let headers: BTreeMap = response.split("\r\n").map(|line| { + let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); + // todo: check there are two parts, and two parts only + // todo: make sure that values from repeated keys get appended within a RadonArray + (String::from(parts[0]), RadonTypes::from(RadonString::from(parts[1]))) + }).collect(); + let input = RadonTypes::from(RadonMap::from(headers)); + let radon_script = unpack_radon_script(&retrieve.script)?; + + execute_radon_script(input, &radon_script, context, settings) +} + /// Handle Rng response with data report fn rng_response_with_data_report( response: &str, @@ -196,7 +216,10 @@ pub fn run_retrieval_with_data_report( RADType::Rng => rng_response_with_data_report(response, context), RADType::HttpPost => { string_response_with_data_report(retrieve, response, context, settings) - } + }, + RADType::HttpHead => { + headers_response_with_data_report(retrieve, response, context, settings) + }, _ => Err(RadError::UnknownRetrieval), } } @@ -214,7 +237,7 @@ pub fn run_retrieval_with_data( .map(RadonReport::into_inner) } -/// Handle generic HTTP (GET/POST) response +/// Handle generic HTTP (GET/POST/HEAD) response async fn http_response( retrieve: &RADRetrieve, context: &mut ReportContext, @@ -258,7 +281,11 @@ async fn http_response( builder.method("POST").uri(&retrieve.url), WitnetHttpBody::from(retrieve.body.clone()), ) - } + }, + RADType::HttpHead => ( + builder.method("HEAD").uri(&retrieve.url), + WitnetHttpBody::empty(), + ), _ => panic!( "Called http_response with invalid retrieval kind {:?}", retrieve.kind @@ -357,6 +384,7 @@ pub async fn run_retrieval_report( RADType::HttpGet => http_response(retrieve, context, settings, client).await, RADType::Rng => rng_response(context, settings).await, RADType::HttpPost => http_response(retrieve, context, settings, client).await, + RADType::HttpHead => http_response(retrieve, context, settings, client).await, _ => Err(RadError::UnknownRetrieval), } } diff --git a/schemas/witnet/witnet.proto b/schemas/witnet/witnet.proto index 64b1b04e0..842009744 100644 --- a/schemas/witnet/witnet.proto +++ b/schemas/witnet/witnet.proto @@ -121,6 +121,7 @@ message DataRequestOutput { HttpGet = 1; Rng = 2; HttpPost = 3; + HttpHead = 4; } message RADFilter { uint32 op = 1; @@ -133,7 +134,7 @@ message DataRequestOutput { bytes script = 3; // Body of HTTP-POST request bytes body = 4; - // Extra headers for HTTP-GET and HTTP-POST requests + // Extra headers for HTTP-GET, HTTP-HEAD and HTTP-POST requests repeated StringPair headers = 5; } message RADAggregate { From 3fdf8f973dcbb168149faafad727cbc084b6ed67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 10 Oct 2023 13:26:31 +0200 Subject: [PATCH 02/46] chore: cargo fmt --all --- data_structures/src/chain/mod.rs | 11 ++++++++--- rad/src/lib.rs | 22 ++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 757f6c46e..7ed2e38e0 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -1652,7 +1652,10 @@ pub enum RADType { impl RADType { pub fn is_http(&self) -> bool { - matches!(self, RADType::HttpGet | RADType::HttpPost | RADType::HttpHead) + matches!( + self, + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead + ) } } @@ -1812,8 +1815,10 @@ impl RADRetrieve { &[Field::Kind, Field::Url, Field::Script], &[Field::Body, Field::Headers], ) - }, - RADType::HttpHead => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) + } + RADType::HttpHead => { + check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) + } } } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index a3781df42..dfa623910 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -181,12 +181,18 @@ fn headers_response_with_data_report( context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let headers: BTreeMap = response.split("\r\n").map(|line| { - let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); - // todo: check there are two parts, and two parts only - // todo: make sure that values from repeated keys get appended within a RadonArray - (String::from(parts[0]), RadonTypes::from(RadonString::from(parts[1]))) - }).collect(); + let headers: BTreeMap = response + .split("\r\n") + .map(|line| { + let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); + // todo: check there are two parts, and two parts only + // todo: make sure that values from repeated keys get appended within a RadonArray + ( + String::from(parts[0]), + RadonTypes::from(RadonString::from(parts[1])), + ) + }) + .collect(); let input = RadonTypes::from(RadonMap::from(headers)); let radon_script = unpack_radon_script(&retrieve.script)?; @@ -216,10 +222,10 @@ pub fn run_retrieval_with_data_report( RADType::Rng => rng_response_with_data_report(response, context), RADType::HttpPost => { string_response_with_data_report(retrieve, response, context, settings) - }, + } RADType::HttpHead => { headers_response_with_data_report(retrieve, response, context, settings) - }, + } _ => Err(RadError::UnknownRetrieval), } } From 721347a1b316e385bdc0225fdc3862d5ff69b6b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:45:08 +0200 Subject: [PATCH 03/46] feat: handle repeated headers within http/head response --- rad/src/lib.rs | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index dfa623910..532a67668 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -181,18 +181,33 @@ fn headers_response_with_data_report( context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let headers: BTreeMap = response - .split("\r\n") - .map(|line| { - let parts: Vec<&str> = line.split(":").map(|part| part.trim()).collect(); - // todo: check there are two parts, and two parts only - // todo: make sure that values from repeated keys get appended within a RadonArray - ( - String::from(parts[0]), - RadonTypes::from(RadonString::from(parts[1])), - ) + let mut headers: BTreeMap> = BTreeMap::new(); + + for line in response.split("\r\n") { + if let Some(first_colon_index) = line.find(":") { + // key: trim spaces and lower case all ascii chars left to the first colon character + let key = String::from(line[0..first_colon_index].trim().to_ascii_lowercase()); + // value: trim spaces on the substring after the first colon character + let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim())); + headers.entry(key).or_default().push(value); + } + } + + let headers: BTreeMap = BTreeMap::from_iter( + headers.iter().map(|(key, value)| { + match value.len() { + len if len > 1 => ( + key.clone(), + RadonTypes::from(RadonArray::from(value.to_vec())) + ), + _ => ( + key.clone(), + value[0].clone() + ) + } }) - .collect(); + ); + let input = RadonTypes::from(RadonMap::from(headers)); let radon_script = unpack_radon_script(&retrieve.script)?; From ef307337cc7b11b16b4c92a41839cbe2493229b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:47:30 +0200 Subject: [PATCH 04/46] test: run valid http head retrieval --- rad/src/lib.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 532a67668..d28a579ae 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -878,6 +878,38 @@ mod tests { use super::*; + #[test] + fn test_run_http_head_retrieval() { + let script_r = Value::Array(vec![ + Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ]), + ]); + let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); + println!("{:?}", packed_script_r); + + let retrieve = RADRetrieve { + kind: RADType::HttpHead, + url: "https://witnet.io/_nuxt/img/dragon_reading.a37f8cb.png".to_string(), + script: packed_script_r, + body: vec![], + headers: vec![], + }; + let response = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 498219\r\nContent-Length: 123456\r\neTag: \"64eca181-79a2b\"\r\n"; + let result = run_retrieval_with_data( + &retrieve, + response, + RadonScriptExecutionSettings::disable_all(), + current_active_wips(), + ).unwrap(); + + match result { + RadonTypes::String(_) => {} + err => panic!("Error in run_retrieval: {:?}", err), + } + } + #[test] fn test_run_retrieval() { let script_r = Value::Array(vec![ From a6653977a7d3aa97da4bf1dcc19d07d239050b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:48:31 +0200 Subject: [PATCH 05/46] test: try http-head data request w/ invalid request header --- rad/src/lib.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index d28a579ae..e07bc4b32 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -302,7 +302,7 @@ async fn http_response( builder.method("POST").uri(&retrieve.url), WitnetHttpBody::from(retrieve.body.clone()), ) - }, + } RADType::HttpHead => ( builder.method("HEAD").uri(&retrieve.url), WitnetHttpBody::empty(), @@ -1607,6 +1607,58 @@ mod tests { } } + #[test] + fn test_try_data_request_http_get_non_ascii_header_key() { + let script_r = Value::Array(vec![]); + let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); + let body = Vec::from(String::from("")); + let headers = vec![("ñ", "value")]; + let headers = headers + .into_iter() + .map(|(a, b)| (a.to_string(), b.to_string())) + .collect(); + let request = RADRequest { + time_lock: 0, + retrieve: vec![RADRetrieve { + kind: RADType::HttpGet, + url: String::from("http://127.0.0.1"), + script: packed_script_r, + body, + headers, + }], + aggregate: RADAggregate { + filters: vec![], + reducer: RadonReducers::Mode as u32, + }, + tally: RADTally { + filters: vec![], + reducer: RadonReducers::Mode as u32, + }, + }; + let report = try_data_request( + &request, + RadonScriptExecutionSettings::enable_all(), + None, + None, + ); + let tally_result = report.tally.into_inner(); + + assert_eq!( + tally_result, + RadonTypes::RadonError( + RadonError::try_from(RadError::UnhandledIntercept { + inner: Some(Box::new(RadError::InvalidHttpHeader { + name: "ñ".to_string(), + value: "value".to_string(), + error: "invalid HTTP header name".to_string() + })), + message: None + }) + .unwrap() + ) + ); + } + #[test] fn test_try_data_request_http_post_non_ascii_header_key() { let script_r = Value::Array(vec![]); From 198e5b8f99f208f2a2662dcb81babc2eacf1d27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:49:09 +0200 Subject: [PATCH 06/46] chore: bump package versions --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- bridges/centralized-ethereum/Cargo.toml | 2 +- data_structures/Cargo.toml | 2 +- node/Cargo.toml | 2 +- rad/Cargo.toml | 2 +- toolkit/Cargo.toml | 2 +- wallet/Cargo.toml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f0d09be9..4571cdfa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4958,7 +4958,7 @@ dependencies = [ [[package]] name = "witnet" -version = "1.6.7" +version = "1.7.0" dependencies = [ "ansi_term", "bytecount", @@ -5005,7 +5005,7 @@ dependencies = [ [[package]] name = "witnet-centralized-ethereum-bridge" -version = "1.6.7" +version = "1.7.0" dependencies = [ "actix", "async-jsonrpc-client", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "witnet_data_structures" -version = "1.6.7" +version = "1.7.0" dependencies = [ "bech32", "bencher", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "witnet_node" -version = "1.6.7" +version = "1.7.0" dependencies = [ "actix", "ansi_term", @@ -5198,7 +5198,7 @@ dependencies = [ [[package]] name = "witnet_rad" -version = "0.3.2" +version = "0.3.3" dependencies = [ "cbor-codec", "failure", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "witnet_toolkit" -version = "1.6.7" +version = "1.7.0" dependencies = [ "failure", "hex", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "witnet_wallet" -version = "1.6.7" +version = "1.7.0" dependencies = [ "actix", "async-jsonrpc-client", diff --git a/Cargo.toml b/Cargo.toml index 0a428c2fb..4a1b64b65 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet" -version = "1.6.7" +version = "1.7.0" authors = ["Witnet Foundation "] publish = false repository = "witnet/witnet-rust" diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml index 5492feb9e..b87084520 100644 --- a/bridges/centralized-ethereum/Cargo.toml +++ b/bridges/centralized-ethereum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet-centralized-ethereum-bridge" -version = "1.6.7" +version = "1.7.0" authors = ["Witnet Foundation "] edition = "2018" diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml index fa801502b..f09cf9699 100644 --- a/data_structures/Cargo.toml +++ b/data_structures/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Witnet Foundation "] description = "data structures component" edition = "2021" name = "witnet_data_structures" -version = "1.6.7" +version = "1.7.0" workspace = ".." [features] diff --git a/node/Cargo.toml b/node/Cargo.toml index 796ca2737..88a31f51a 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_node" -version = "1.6.7" +version = "1.7.0" authors = ["Witnet Foundation "] workspace = ".." description = "node component" diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 806b62527..d9bef65f3 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_rad" -version = "0.3.2" +version = "0.3.3" authors = ["Witnet Foundation "] edition = "2021" workspace = ".." diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml index eb4a5096d..5ee64aecd 100644 --- a/toolkit/Cargo.toml +++ b/toolkit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_toolkit" -version = "1.6.7" +version = "1.7.0" authors = ["Adán SDPC "] edition = "2021" diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml index 4bf3bb766..ec59fb12b 100644 --- a/wallet/Cargo.toml +++ b/wallet/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Witnet Foundation "] edition = "2021" name = "witnet_wallet" -version = "1.6.7" +version = "1.7.0" workspace = ".." [dependencies] From 9bae448eb61a266b2fe36ebe6386fcd8a940e08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 10:52:59 +0200 Subject: [PATCH 07/46] chore: cargo fmt --all --- rad/src/lib.rs | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index e07bc4b32..d21e663ca 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -192,21 +192,15 @@ fn headers_response_with_data_report( headers.entry(key).or_default().push(value); } } - - let headers: BTreeMap = BTreeMap::from_iter( - headers.iter().map(|(key, value)| { - match value.len() { - len if len > 1 => ( - key.clone(), - RadonTypes::from(RadonArray::from(value.to_vec())) - ), - _ => ( - key.clone(), - value[0].clone() - ) - } - }) - ); + + let headers: BTreeMap = + BTreeMap::from_iter(headers.iter().map(|(key, value)| match value.len() { + len if len > 1 => ( + key.clone(), + RadonTypes::from(RadonArray::from(value.to_vec())), + ), + _ => (key.clone(), value[0].clone()), + })); let input = RadonTypes::from(RadonMap::from(headers)); let radon_script = unpack_radon_script(&retrieve.script)?; @@ -880,12 +874,10 @@ mod tests { #[test] fn test_run_http_head_retrieval() { - let script_r = Value::Array(vec![ - Value::Array(vec![ - Value::Integer(RadonOpCodes::MapGetString as i128), - Value::Text("etag".to_string()), - ]), - ]); + let script_r = Value::Array(vec![Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ])]); let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); println!("{:?}", packed_script_r); @@ -902,7 +894,8 @@ mod tests { response, RadonScriptExecutionSettings::disable_all(), current_active_wips(), - ).unwrap(); + ) + .unwrap(); match result { RadonTypes::String(_) => {} From af5ff8643a735a534fd322dc9423e5b4c9668cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 11:59:33 +0200 Subject: [PATCH 08/46] chore: solve clippy warnings --- rad/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index d21e663ca..5fa1e8979 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -184,9 +184,9 @@ fn headers_response_with_data_report( let mut headers: BTreeMap> = BTreeMap::new(); for line in response.split("\r\n") { - if let Some(first_colon_index) = line.find(":") { + if let Some(first_colon_index) = line.find(':') { // key: trim spaces and lower case all ascii chars left to the first colon character - let key = String::from(line[0..first_colon_index].trim().to_ascii_lowercase()); + let key = line[0..first_colon_index].trim().to_ascii_lowercase(); // value: trim spaces on the substring after the first colon character let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim())); headers.entry(key).or_default().push(value); @@ -692,7 +692,7 @@ fn validate_header(name: &str, value: &str) -> Result<()> { Err(RadError::InvalidHttpHeader { name: name.to_string(), value: value.to_string(), - error: error_message.to_string(), + error: error_message, }) } else { Ok(()) From ac57deddef050bba326bce951bc6f49ca592afb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 11 Oct 2023 17:51:28 +0200 Subject: [PATCH 09/46] fix: use http::Response::headers(&self) to transform response to HTTP/HEAD into a JSON string Instead or parsing http::Response::body into a RadonMap --- net/src/client/http/mod.rs | 2 +- rad/src/lib.rs | 84 +++++++++++++------------------------- 2 files changed, 29 insertions(+), 57 deletions(-) diff --git a/net/src/client/http/mod.rs b/net/src/client/http/mod.rs index 55ae33e9e..3fca5533b 100644 --- a/net/src/client/http/mod.rs +++ b/net/src/client/http/mod.rs @@ -151,7 +151,7 @@ pub struct WitnetHttpResponse { impl WitnetHttpResponse { #[inline] - /// Simple wrapper around `isahc::Response::status`. + /// Simple wrapper around `isahc::Response`. pub fn inner(self) -> isahc::Response { self.res } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 5fa1e8979..3c3990dea 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -25,11 +25,10 @@ use crate::{ create_radon_script_from_filters_and_reducer, execute_radon_script, unpack_radon_script, RadonScriptExecutionSettings, }, - types::{array::RadonArray, bytes::RadonBytes, map::RadonMap, string::RadonString, RadonTypes}, + types::{array::RadonArray, bytes::RadonBytes, string::RadonString, RadonTypes}, user_agents::UserAgent, }; use core::convert::From; -use std::collections::BTreeMap; use witnet_net::client::http::{WitnetHttpBody, WitnetHttpRequest}; pub mod conditions; @@ -174,40 +173,6 @@ fn string_response_with_data_report( execute_radon_script(input, &radon_script, context, settings) } -/// Handle HTTP-HEAD response with data, and return a `RadonReport`. -fn headers_response_with_data_report( - retrieve: &RADRetrieve, - response: &str, - context: &mut ReportContext, - settings: RadonScriptExecutionSettings, -) -> Result> { - let mut headers: BTreeMap> = BTreeMap::new(); - - for line in response.split("\r\n") { - if let Some(first_colon_index) = line.find(':') { - // key: trim spaces and lower case all ascii chars left to the first colon character - let key = line[0..first_colon_index].trim().to_ascii_lowercase(); - // value: trim spaces on the substring after the first colon character - let value = RadonTypes::from(RadonString::from(line[first_colon_index + 1..].trim())); - headers.entry(key).or_default().push(value); - } - } - - let headers: BTreeMap = - BTreeMap::from_iter(headers.iter().map(|(key, value)| match value.len() { - len if len > 1 => ( - key.clone(), - RadonTypes::from(RadonArray::from(value.to_vec())), - ), - _ => (key.clone(), value[0].clone()), - })); - - let input = RadonTypes::from(RadonMap::from(headers)); - let radon_script = unpack_radon_script(&retrieve.script)?; - - execute_radon_script(input, &radon_script, context, settings) -} - /// Handle Rng response with data report fn rng_response_with_data_report( response: &str, @@ -227,14 +192,10 @@ pub fn run_retrieval_with_data_report( settings: RadonScriptExecutionSettings, ) -> Result> { match retrieve.kind { - RADType::HttpGet => string_response_with_data_report(retrieve, response, context, settings), - RADType::Rng => rng_response_with_data_report(response, context), - RADType::HttpPost => { + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead => { string_response_with_data_report(retrieve, response, context, settings) } - RADType::HttpHead => { - headers_response_with_data_report(retrieve, response, context, settings) - } + RADType::Rng => rng_response_with_data_report(response, context), _ => Err(RadError::UnknownRetrieval), } } @@ -340,13 +301,21 @@ async fn http_response( // If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is // where we need to decide how to read the response body - let (_parts, mut body) = response.into_parts(); let mut response_string = String::default(); - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; + + let (parts, mut body) = response.into_parts(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; + } + } let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings); @@ -396,10 +365,10 @@ pub async fn run_retrieval_report( context.set_active_wips(active_wips); match retrieve.kind { - RADType::HttpGet => http_response(retrieve, context, settings, client).await, + RADType::HttpGet | RADType::HttpHead | RADType::HttpPost => { + http_response(retrieve, context, settings, client).await + } RADType::Rng => rng_response(context, settings).await, - RADType::HttpPost => http_response(retrieve, context, settings, client).await, - RADType::HttpHead => http_response(retrieve, context, settings, client).await, _ => Err(RadError::UnknownRetrieval), } } @@ -874,10 +843,13 @@ mod tests { #[test] fn test_run_http_head_retrieval() { - let script_r = Value::Array(vec![Value::Array(vec![ - Value::Integer(RadonOpCodes::MapGetString as i128), - Value::Text("etag".to_string()), - ])]); + let script_r = Value::Array(vec![ + Value::Integer(RadonOpCodes::StringParseJSONMap as i128), + Value::Array(vec![ + Value::Integer(RadonOpCodes::MapGetString as i128), + Value::Text("etag".to_string()), + ]), + ]); let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); println!("{:?}", packed_script_r); @@ -888,7 +860,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 498219\r\nContent-Length: 123456\r\neTag: \"64eca181-79a2b\"\r\n"; + let response = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; let result = run_retrieval_with_data( &retrieve, response, From 5e073f9ac26d577bc5cdbe4f619039f1f280ef6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 20 Oct 2023 09:52:56 +0200 Subject: [PATCH 10/46] chore: attend pr review comments --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- bridges/centralized-ethereum/Cargo.toml | 2 +- data_structures/Cargo.toml | 2 +- node/Cargo.toml | 2 +- rad/src/lib.rs | 3 +-- toolkit/Cargo.toml | 2 +- wallet/Cargo.toml | 2 +- 8 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4571cdfa6..8580a36aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4958,7 +4958,7 @@ dependencies = [ [[package]] name = "witnet" -version = "1.7.0" +version = "2.0.0" dependencies = [ "ansi_term", "bytecount", @@ -5005,7 +5005,7 @@ dependencies = [ [[package]] name = "witnet-centralized-ethereum-bridge" -version = "1.7.0" +version = "2.0.0" dependencies = [ "actix", "async-jsonrpc-client", @@ -5069,7 +5069,7 @@ dependencies = [ [[package]] name = "witnet_data_structures" -version = "1.7.0" +version = "2.0.0" dependencies = [ "bech32", "bencher", @@ -5134,7 +5134,7 @@ dependencies = [ [[package]] name = "witnet_node" -version = "1.7.0" +version = "2.0.0" dependencies = [ "actix", "ansi_term", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "witnet_toolkit" -version = "1.7.0" +version = "2.0.0" dependencies = [ "failure", "hex", @@ -5284,7 +5284,7 @@ dependencies = [ [[package]] name = "witnet_wallet" -version = "1.7.0" +version = "2.0.0" dependencies = [ "actix", "async-jsonrpc-client", diff --git a/Cargo.toml b/Cargo.toml index 4a1b64b65..21a934c11 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet" -version = "1.7.0" +version = "2.0.0" authors = ["Witnet Foundation "] publish = false repository = "witnet/witnet-rust" diff --git a/bridges/centralized-ethereum/Cargo.toml b/bridges/centralized-ethereum/Cargo.toml index b87084520..6e3a4f7a0 100644 --- a/bridges/centralized-ethereum/Cargo.toml +++ b/bridges/centralized-ethereum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet-centralized-ethereum-bridge" -version = "1.7.0" +version = "2.0.0" authors = ["Witnet Foundation "] edition = "2018" diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml index f09cf9699..9fd896c71 100644 --- a/data_structures/Cargo.toml +++ b/data_structures/Cargo.toml @@ -3,7 +3,7 @@ authors = ["Witnet Foundation "] description = "data structures component" edition = "2021" name = "witnet_data_structures" -version = "1.7.0" +version = "2.0.0" workspace = ".." [features] diff --git a/node/Cargo.toml b/node/Cargo.toml index 88a31f51a..57fe55a7a 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_node" -version = "1.7.0" +version = "2.0.0" authors = ["Witnet Foundation "] workspace = ".." description = "node component" diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 3c3990dea..e7eb65541 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -851,11 +851,10 @@ mod tests { ]), ]); let packed_script_r = serde_cbor::to_vec(&script_r).unwrap(); - println!("{:?}", packed_script_r); let retrieve = RADRetrieve { kind: RADType::HttpHead, - url: "https://witnet.io/_nuxt/img/dragon_reading.a37f8cb.png".to_string(), + url: "https://en.wikipedia.org/static/images/icons/wikipedia.png".to_string(), script: packed_script_r, body: vec![], headers: vec![], diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml index 5ee64aecd..7ac7b72d4 100644 --- a/toolkit/Cargo.toml +++ b/toolkit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "witnet_toolkit" -version = "1.7.0" +version = "2.0.0" authors = ["Adán SDPC "] edition = "2021" diff --git a/wallet/Cargo.toml b/wallet/Cargo.toml index ec59fb12b..5a7221a5f 100644 --- a/wallet/Cargo.toml +++ b/wallet/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Witnet Foundation "] edition = "2021" name = "witnet_wallet" -version = "1.7.0" +version = "2.0.0" workspace = ".." [dependencies] From 6c4d1ac2006da442b55fa12cd2b8f7991be44547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 20 Oct 2023 12:44:25 +0200 Subject: [PATCH 11/46] feat(rad): add support to binary sources --- rad/src/lib.rs | 88 ++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index e7eb65541..6f9b8f693 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -78,7 +78,12 @@ pub fn try_data_request( .iter() .zip(inputs.iter()) .map(|(retrieve, input)| { - run_retrieval_with_data_report(retrieve, input, &mut retrieval_context, settings) + run_retrieval_with_data_report( + retrieve, + RadonTypes::from(RadonString::from(*input)), + &mut retrieval_context, + settings, + ) }) .collect() } else { @@ -160,42 +165,28 @@ pub fn try_data_request( } } -/// Handle HTTP-GET and HTTP-POST response with data, and return a `RadonReport`. -fn string_response_with_data_report( +/// Execute Radon Script using as input the RadonTypes value deserialized from a retrieval response +fn handle_response_with_data_report( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { - let input = RadonTypes::from(RadonString::from(response)); let radon_script = unpack_radon_script(&retrieve.script)?; - - execute_radon_script(input, &radon_script, context, settings) -} - -/// Handle Rng response with data report -fn rng_response_with_data_report( - response: &str, - context: &mut ReportContext, -) -> Result> { - let response_bytes = response.as_bytes(); - let result = RadonTypes::from(RadonBytes::from(response_bytes.to_vec())); - - Ok(RadonReport::from_result(Ok(result), context)) + execute_radon_script(response, &radon_script, context, settings) } /// Run retrieval without performing any external network requests, return `Result`. pub fn run_retrieval_with_data_report( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, context: &mut ReportContext, settings: RadonScriptExecutionSettings, ) -> Result> { match retrieve.kind { - RADType::HttpGet | RADType::HttpPost | RADType::HttpHead => { - string_response_with_data_report(retrieve, response, context, settings) + RADType::HttpGet | RADType::HttpPost | RADType::HttpHead | RADType::Rng => { + handle_response_with_data_report(retrieve, response, context, settings) } - RADType::Rng => rng_response_with_data_report(response, context), _ => Err(RadError::UnknownRetrieval), } } @@ -203,7 +194,7 @@ pub fn run_retrieval_with_data_report( /// Run retrieval without performing any external network requests, return `Result`. pub fn run_retrieval_with_data( retrieve: &RADRetrieve, - response: &str, + response: RadonTypes, settings: RadonScriptExecutionSettings, active_wips: ActiveWips, ) -> Result { @@ -299,26 +290,48 @@ async fn http_response( }); } - // If at some point we want to support the retrieval of non-UTF8 data (e.g. raw bytes), this is - // where we need to decide how to read the response body - let mut response_string = String::default(); - let (parts, mut body) = response.into_parts(); - match retrieve.kind { - RADType::HttpHead => { - response_string = format!("{:?}", parts.headers); + + let response: RadonTypes; + match parts.headers.get("accept-ranges") { + Some(_) => { + // http response is a binary stream + let mut response_bytes = Vec::::default(); + match retrieve.kind { + RADType::HttpHead => { + // todo: assert http-head responses should never return binary streams + } + _ => { + // todo: before reading the response buffer, an error should thrown it was too big + body.read_to_end(&mut response_bytes).await.map_err(|x| { + RadError::HttpOther { + message: x.to_string(), + } + })?; + } + } + response = RadonTypes::from(RadonBytes::from(response_bytes)); } _ => { - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; + // response is a string + let mut response_string = String::default(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; + } + } + response = RadonTypes::from(RadonString::from(response_string)); } } - let result = run_retrieval_with_data_report(retrieve, &response_string, context, settings); - + let result = handle_response_with_data_report(retrieve, response, context, settings); match &result { Ok(report) => { log::debug!( @@ -329,7 +342,6 @@ async fn http_response( } Err(e) => log::debug!("Failed result for source {}: {:?}", retrieve.url, e), } - result } From 0cfd81d4aad31b2212ee0b354fc45b6175d8940a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 20 Oct 2023 12:45:00 +0200 Subject: [PATCH 12/46] chore(rad): refactor existing tests --- rad/src/lib.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 6f9b8f693..2af893083 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -871,10 +871,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; + let response_string = r#"{"date": "Wed, 11 Oct 2023 15:18:42 GMT", "content-type": "image/png", "content-length": "498219", "x-origin-cache": "HIT", "last-modified": "Mon, 28 Aug 2023 13:30:41 GMT", "access-control-allow-origin": "*", "etag": "\"64eca181-79a2b\"", "expires": "Wed, 11 Oct 2023 15:28:41 GMT", "cache-control": "max-age=1800", "x-proxy-cache": "MISS", "x-github-request-id": "6750:35DB:BF8211:FEFD2B:652602FA", "via": "1.1 varnish", "x-served-by": "cache-hnd18736-HND", "x-cache": "MISS", "x-cache-hits": "0", "x-timer": "S1696989946.496383,VS0,VE487", "vary": "Accept-Encoding", "x-fastly-request-id": "118bdfd8a926cbdc781bc23079c3dc07a22d2223", "cf-cache-status": "REVALIDATED", "accept-ranges": "bytes", "report-to": "{\"endpoints\":[{\"url\":\"https:\/\/a.nel.cloudflare.com\/report\/v3?s=FlzxKRCYYN4SL0x%2FraG7ugKCqdC%2BeQqVrucvsfeDWf%2F7A0Nv9fv7TYRgU0WL4k1kbZyxt%2B04VjOyv0XK55sF37GEPwXHE%2FdXnoFlWutID762k2ktcX6hUml6oNk%3D\"}],\"group\":\"cf-nel\",\"max_age\":604800}", "nel": "{\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}", "strict-transport-security": "max-age=0", "x-content-type-options": "nosniff", "server": "cloudflare", "cf-ray": "814813bf3a73f689-NRT", "alt-svc": "h3=\":443\"; ma=86400"}"#; let result = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -908,11 +908,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#; - + let response_string = r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":500,"main":"Rain","description":"light rain","icon":"10d"}],"base":"stations","main":{"temp":17.59,"pressure":1022,"humidity":67,"temp_min":15,"temp_max":20},"visibility":10000,"wind":{"speed":3.6,"deg":260},"rain":{"1h":0.51},"clouds":{"all":20},"dt":1567501321,"sys":{"type":1,"id":1275,"message":0.0089,"country":"DE","sunrise":1567484402,"sunset":1567533129},"timezone":7200,"id":2950159,"name":"Berlin","cod":200}"#; let result = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -967,7 +966,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "84"; + let response_string = "84"; let expected = RadonTypes::Float(RadonFloat::from(84)); let aggregate = RADAggregate { @@ -982,7 +981,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1005,7 +1004,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = "307"; + let response_string = "307"; let expected = RadonTypes::Float(RadonFloat::from(307)); let aggregate = RADAggregate { @@ -1019,7 +1018,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1058,7 +1057,7 @@ mod tests { headers: vec![], }; // This response was modified because the original was about 100KB. - let response = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#; + let response_string = r#"[{"estacion_nombre":"Pza. de España","estacion_numero":4,"fecha":"03092019","hora0":{"estado":"Pasado","valor":"00008"}}]"#; let expected = RadonTypes::Float(RadonFloat::from(8)); let aggregate = RADAggregate { @@ -1072,7 +1071,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1102,7 +1101,7 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#; + let response_string = r#"{"PSOE":123,"PP":66,"Cs":57,"UP":42,"VOX":24,"ERC-SOBIRANISTES":15,"JxCAT-JUNTS":7,"PNV":6,"EH Bildu":4,"CCa-PNC":2,"NA+":2,"COMPROMÍS 2019":1,"PRC":1,"PACMA":0,"FRONT REPUBLICÀ":0,"BNG":0,"RECORTES CERO-GV":0,"NCa":0,"PACT":0,"ARA-MES-ESQUERRA":0,"GBAI":0,"PUM+J":0,"EN MAREA":0,"PCTE":0,"EL PI":0,"AxSI":0,"PCOE":0,"PCPE":0,"AVANT ADELANTE LOS VERDES":0,"EB":0,"CpM":0,"SOMOS REGIÓN":0,"PCPA":0,"PH":0,"UIG-SOM-CUIDES":0,"ERPV":0,"IZQP":0,"PCPC":0,"AHORA CANARIAS":0,"CxG":0,"PPSO":0,"CNV":0,"PREPAL":0,"C.Ex-C.R.Ex-P.R.Ex":0,"PR+":0,"P-LIB":0,"CILU-LINARES":0,"ANDECHA ASTUR":0,"JF":0,"PYLN":0,"FIA":0,"FE de las JONS":0,"SOLIDARIA":0,"F8":0,"DPL":0,"UNIÓN REGIONALISTA":0,"centrados":0,"DP":0,"VOU":0,"PDSJE-UDEC":0,"IZAR":0,"RISA":0,"C 21":0,"+MAS+":0,"UDT":0}"#; let expected = RadonTypes::Float(RadonFloat::from(123)); let aggregate = RADAggregate { @@ -1116,7 +1115,7 @@ mod tests { let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) @@ -1157,10 +1156,10 @@ mod tests { body: vec![], headers: vec![], }; - let response = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#; + let response_string = r#"{"event":{"homeTeam":{"name":"Ryazan-VDV","slug":"ryazan-vdv","gender":"F","national":false,"id":171120,"shortName":"Ryazan-VDV","subTeams":[]},"awayTeam":{"name":"Olympique Lyonnais","slug":"olympique-lyonnais","gender":"F","national":false,"id":26245,"shortName":"Lyon","subTeams":[]},"homeScore":{"current":0,"display":0,"period1":0,"normaltime":0},"awayScore":{"current":9,"display":9,"period1":5,"normaltime":9}}}"#; let retrieved = run_retrieval_with_data( &retrieve, - response, + RadonTypes::from(RadonString::from(response_string)), RadonScriptExecutionSettings::disable_all(), current_active_wips(), ) From 68c7648180863c17c4271cdbeaca9a8a9ed344d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Thu, 26 Oct 2023 13:24:08 +0200 Subject: [PATCH 13/46] chore: attend pr review comments --- rad/src/lib.rs | 68 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 2af893083..a30cd0309 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -13,7 +13,7 @@ use witnet_data_structures::{ RADAggregate, RADRequest, RADRetrieve, RADTally, RADType, }, radon_report::{RadonReport, ReportContext, RetrievalMetadata, Stage, TallyMetaData}, - witnessing::WitnessingConfig, + witnessing::WitnessingConfig, radon_error::RadonError, }; use witnet_net::client::http::WitnetHttpClient; pub use witnet_net::Uri; @@ -293,42 +293,44 @@ async fn http_response( let (parts, mut body) = response.into_parts(); let response: RadonTypes; - match parts.headers.get("accept-ranges") { - Some(_) => { - // http response is a binary stream - let mut response_bytes = Vec::::default(); - match retrieve.kind { - RADType::HttpHead => { - // todo: assert http-head responses should never return binary streams - } - _ => { - // todo: before reading the response buffer, an error should thrown it was too big - body.read_to_end(&mut response_bytes).await.map_err(|x| { - RadError::HttpOther { - message: x.to_string(), - } - })?; - } + if parts.headers.contains_key("accept-ranges") { + // http response is a binary stream + let mut response_bytes = Vec::::default(); + match retrieve.kind { + RADType::HttpHead => { + response = RadonTypes::RadonError( + RadonError::try_from(RadError::BufferIsNotValue { + description: String::from("Unsupported binary streams from HTTP/HEAD sources") + }) + .unwrap() + ); + } + _ => { + // todo: before reading the response buffer, an error should be thrown if it was too big + body.read_to_end(&mut response_bytes).await.map_err(|x| { + RadError::HttpOther { + message: x.to_string(), + } + })?; + response = RadonTypes::from(RadonBytes::from(response_bytes)); } - response = RadonTypes::from(RadonBytes::from(response_bytes)); } - _ => { - // response is a string - let mut response_string = String::default(); - match retrieve.kind { - RADType::HttpHead => { - response_string = format!("{:?}", parts.headers); - } - _ => { - body.read_to_string(&mut response_string) - .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), - })?; - } + } else { + // response is a string + let mut response_string = String::default(); + match retrieve.kind { + RADType::HttpHead => { + response_string = format!("{:?}", parts.headers); + } + _ => { + body.read_to_string(&mut response_string) + .await + .map_err(|x| RadError::HttpOther { + message: x.to_string(), + })?; } - response = RadonTypes::from(RadonString::from(response_string)); } + response = RadonTypes::from(RadonString::from(response_string)); } let result = handle_response_with_data_report(retrieve, response, context, settings); From 184f768850a16a201679fc25f67b051f7d415f0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Thu, 26 Oct 2023 13:27:44 +0200 Subject: [PATCH 14/46] chore: cargo clippy --fix --- data_structures/src/chain/mod.rs | 4 ++-- data_structures/src/data_request.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 7ed2e38e0..8b9b3d729 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -2690,7 +2690,7 @@ impl TransactionsPool { for input in &vt_tx.body.inputs { self.output_pointer_map .entry(input.output_pointer) - .or_insert_with(Vec::new) + .or_default() .push(vt_tx.hash()); } @@ -2715,7 +2715,7 @@ impl TransactionsPool { for input in &dr_tx.body.inputs { self.output_pointer_map .entry(input.output_pointer) - .or_insert_with(Vec::new) + .or_default() .push(dr_tx.hash()); } diff --git a/data_structures/src/data_request.rs b/data_structures/src/data_request.rs index 6dd3493a1..fc3823037 100644 --- a/data_structures/src/data_request.rs +++ b/data_structures/src/data_request.rs @@ -143,7 +143,7 @@ impl DataRequestPool { self.data_requests_by_epoch .entry(epoch) - .or_insert_with(HashSet::new) + .or_default() .insert(dr_hash); self.data_request_pool.insert(dr_hash, dr_state); From cd4641c84aae3593d2395dd3f63147c7875a2f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Thu, 26 Oct 2023 19:14:34 +0200 Subject: [PATCH 15/46] feat(rad): add new RadonErrors::BufferIsNotValue --- data_structures/src/radon_error.rs | 2 ++ rad/src/error.rs | 5 +++++ rad/src/lib.rs | 13 +++++-------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs index 7d3176629..6fdf30d26 100644 --- a/data_structures/src/radon_error.rs +++ b/data_structures/src/radon_error.rs @@ -34,6 +34,8 @@ pub enum RadonErrors { HTTPError = 0x30, /// Al least one of the sources could not be retrieved, timeout reached. RetrieveTimeout = 0x31, + /// Value cannot be extracted from binary buffer + BufferIsNotValue = 0x32, // Math errors /// Math operator caused an underflow. Underflow = 0x40, diff --git a/rad/src/error.rs b/rad/src/error.rs index a413f47c4..b9f2086b6 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -431,6 +431,10 @@ impl RadError { } Ok(RadonError::new(match kind { + RadonErrors::BufferIsNotValue => { + let (description,) = deserialize_args(error_args)?; + RadError::BufferIsNotValue { description } + } RadonErrors::RequestTooManySources => RadError::RequestTooManySources, RadonErrors::ScriptTooManyCalls => RadError::ScriptTooManyCalls, RadonErrors::Overflow => RadError::Overflow, @@ -574,6 +578,7 @@ impl RadError { pub fn try_into_error_code(&self) -> Result { Ok(match self { RadError::Unknown => RadonErrors::Unknown, + RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue, RadError::SourceScriptNotCBOR => RadonErrors::SourceScriptNotCBOR, RadError::SourceScriptNotArray => RadonErrors::SourceScriptNotArray, RadError::SourceScriptNotRADON => RadonErrors::SourceScriptNotRADON, diff --git a/rad/src/lib.rs b/rad/src/lib.rs index a30cd0309..5de2f1379 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -13,7 +13,7 @@ use witnet_data_structures::{ RADAggregate, RADRequest, RADRetrieve, RADTally, RADType, }, radon_report::{RadonReport, ReportContext, RetrievalMetadata, Stage, TallyMetaData}, - witnessing::WitnessingConfig, radon_error::RadonError, + witnessing::WitnessingConfig, }; use witnet_net::client::http::WitnetHttpClient; pub use witnet_net::Uri; @@ -298,12 +298,9 @@ async fn http_response( let mut response_bytes = Vec::::default(); match retrieve.kind { RADType::HttpHead => { - response = RadonTypes::RadonError( - RadonError::try_from(RadError::BufferIsNotValue { - description: String::from("Unsupported binary streams from HTTP/HEAD sources") - }) - .unwrap() - ); + return Err(RadError::BufferIsNotValue { + description: String::from("Unsupported binary streams from HTTP/HEAD sources") + }); } _ => { // todo: before reading the response buffer, an error should be thrown if it was too big @@ -312,9 +309,9 @@ async fn http_response( message: x.to_string(), } })?; - response = RadonTypes::from(RadonBytes::from(response_bytes)); } } + response = RadonTypes::from(RadonBytes::from(response_bytes)); } else { // response is a string let mut response_string = String::default(); From f1314a4626a1f47bad478b0d4d6964897ba9a689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:36:02 +0100 Subject: [PATCH 16/46] fix(rad): http-head response headers can contain 'accept-ranges' --- rad/src/lib.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 5de2f1379..289178f98 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -293,24 +293,16 @@ async fn http_response( let (parts, mut body) = response.into_parts(); let response: RadonTypes; - if parts.headers.contains_key("accept-ranges") { + if retrieve.kind != RADType::HttpHead && parts.headers.contains_key("accept-ranges") { // http response is a binary stream let mut response_bytes = Vec::::default(); - match retrieve.kind { - RADType::HttpHead => { - return Err(RadError::BufferIsNotValue { - description: String::from("Unsupported binary streams from HTTP/HEAD sources") - }); - } - _ => { - // todo: before reading the response buffer, an error should be thrown if it was too big - body.read_to_end(&mut response_bytes).await.map_err(|x| { - RadError::HttpOther { - message: x.to_string(), - } - })?; + + // todo: before reading the response buffer, an error should be thrown if it was too big + body.read_to_end(&mut response_bytes).await.map_err(|x| { + RadError::HttpOther { + message: x.to_string(), } - } + })?; response = RadonTypes::from(RadonBytes::from(response_bytes)); } else { // response is a string From 1f2402b1b953e94e54ae5fb9e07038f765aa8951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:39:29 +0100 Subject: [PATCH 17/46] feat(rad): implement StringReplace --- rad/src/operators/mod.rs | 1 + rad/src/operators/string.rs | 13 +++++++++++++ rad/src/types/string.rs | 3 +++ 3 files changed, 17 insertions(+) diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 6964f2623..1ccceed8f 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -106,6 +106,7 @@ pub enum RadonOpCodes { StringParseXMLMap = 0x78, StringToLowerCase = 0x79, StringToUpperCase = 0x7A, + StringReplace = 0x7B, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index e23914e99..a5dd6197b 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -7,6 +7,8 @@ use std::{ use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; +use regex::Regex; + use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, @@ -224,6 +226,17 @@ pub fn hash(input: &RadonString, args: &[Value]) -> Result Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringReplace".to_string(), + args: args.to_vec(), + }; + let regex = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; + let replacement = RadonString::try_from(args.get(1).ok_or_else(wrong_args)?.to_owned())?; + Ok(RadonString::from(input.value().as_str().replace(regex.value().as_str(), replacement.value().as_str()))) +} + pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index 16eb9ccf3..fb42ad9b5 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -143,6 +143,9 @@ impl Operable for RadonString { (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) .map(RadonTypes::from) .map_err(Into::into), + (RadonOpCodes::StringReplace, Some(args)) => { + string_operators::string_replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), From 533c10367b7cd788e22163d8425b5252358a89ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:42:44 +0100 Subject: [PATCH 18/46] feat(rad): implement StringSlice --- rad/Cargo.toml | 1 + rad/src/operators/mod.rs | 1 + rad/src/operators/string.rs | 22 ++++++++++++++++++++++ rad/src/types/string.rs | 3 +++ 4 files changed, 27 insertions(+) diff --git a/rad/Cargo.toml b/rad/Cargo.toml index d9bef65f3..965e21105 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -26,6 +26,7 @@ rand = "0.7.3" serde = "1.0.111" serde_cbor = "0.11.2" serde_json = "1.0.96" +slicestring = "0.3.2" # the url crate is used to perform additional validations before passing arguments to the surf http client # the version of url must be kept in sync with the version used by surf in the `witnet_net` crate url = "2.1.1" diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 1ccceed8f..5f483f280 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -107,6 +107,7 @@ pub enum RadonOpCodes { StringToLowerCase = 0x79, StringToUpperCase = 0x7A, StringReplace = 0x7B, + StringSlice = 0x7C, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index a5dd6197b..f7560958a 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -7,6 +7,7 @@ use std::{ use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; +use slicestring::Slice; use regex::Regex; use crate::{ @@ -237,6 +238,27 @@ pub fn string_replace(input: &RadonString, args: &[Value]) -> Result Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringSlice".to_string(), + args: args.to_vec(), + }; + let mut end_index: usize = input.value().len(); + match args.len() { + 2 => { + let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + end_index = from_value::(args[1].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + Ok(RadonString::from(input.value().as_str().slice(start_index..end_index))) + } + 1 => { + let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + Ok(RadonString::from(input.value().as_str().slice(start_index..end_index))) + } + _ => Err(wrong_args()) + } +} + pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index fb42ad9b5..fce29c114 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -146,6 +146,9 @@ impl Operable for RadonString { (RadonOpCodes::StringReplace, Some(args)) => { string_operators::string_replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } + (RadonOpCodes::StringSlice, Some(args)) => { + string_operators::string_slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), From b3322b1bdc795f4cf8590de5784ad097c990178d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:43:54 +0100 Subject: [PATCH 19/46] feat(rad): implement StringSplit --- Cargo.lock | 8 ++++++++ rad/Cargo.toml | 1 + rad/src/operators/mod.rs | 1 + rad/src/operators/string.rs | 13 ++++++++++++- rad/src/types/string.rs | 3 +++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8580a36aa..de24156e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3719,6 +3719,12 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "slicestring" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7b0ad4ee74da25b3ff80bc5aa633bf8b1e0bbaf577c6f44529a26ad2d9df75" + [[package]] name = "sluice" version = "0.5.5" @@ -5211,9 +5217,11 @@ dependencies = [ "num_enum", "ordered-float", "rand 0.7.3", + "regex", "serde", "serde_cbor", "serde_json", + "slicestring", "url", "witnet_config", "witnet_crypto", diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 965e21105..2a0d478de 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -23,6 +23,7 @@ minidom = { git = "https://github.com/witnet/xmpp-rs", rev = "bc8a33ff5da95ee403 num_enum = "0.4.2" ordered-float = "3.0" rand = "0.7.3" +regex = "1.4.2" serde = "1.0.111" serde_cbor = "0.11.2" serde_json = "1.0.96" diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 5f483f280..32d6961c1 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -108,6 +108,7 @@ pub enum RadonOpCodes { StringToUpperCase = 0x7A, StringReplace = 0x7B, StringSlice = 0x7C, + StringSplit = 0x7D, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index f7560958a..3149a9cc7 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -259,10 +259,21 @@ pub fn string_slice(input: &RadonString, args: &[Value]) -> Result Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "StringSplit".to_string(), + args: args.to_vec(), + }; + let pattern = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; + let parts: Vec = Regex::new(pattern.value().as_str()).unwrap().split(input.value().as_str()).map(|part| RadonTypes::from(RadonString::from(part))).collect(); + Ok(RadonArray::from(parts)) +} + pub fn string_match(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), - operator: "String match".to_string(), + operator: "StringMatch".to_string(), args: args.to_vec(), }; diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index fce29c114..d12fa79bd 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -149,6 +149,9 @@ impl Operable for RadonString { (RadonOpCodes::StringSlice, Some(args)) => { string_operators::string_slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } + (RadonOpCodes::StringSplit, Some(args)) => { + string_operators::string_split(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), operator: op_code.to_string(), From 88c867496d8414466a21625b2442d290250580ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Tue, 14 Nov 2023 10:45:18 +0100 Subject: [PATCH 20/46] feat(rad): first approach to ArrayJoin --- rad/src/operators/array.rs | 34 ++++++++++++++++++++++++++++++++++ rad/src/operators/mod.rs | 2 +- rad/src/types/array.rs | 1 + 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index b4aac9674..c86f7778e 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -109,6 +109,40 @@ fn get_numeric_string(input: &RadonArray, args: &[Value]) -> Result Result { + // Join not applicable if the input array is not homogeneous + if !input.is_homogeneous() { + return Err(RadError::UnsupportedOpNonHomogeneous { + operator: "ArrayJoin".to_string(), + }); + } + let separator = if args.len() > 0 { + from_value::(args[0].to_owned()).unwrap_or_default() + } else { + String::from("") + }; + match input.value().first() { + Some(RadonTypes::String(_)) => { + let string_list: Vec = input.value().into_iter().map(|item| + RadonString::try_from(item).unwrap_or_default().value() + ).collect(); + Ok(RadonTypes::from(RadonString::from(string_list.join(separator.as_str())))) + Some(first_item) => { + return Err(RadError::UnsupportedOperator { + input_type: first_item.radon_type_name().to_string(), + operator: "ArrayJoin".to_string(), + args: Some(args.to_vec()) + }); + } + _ => { + return Err(RadError::EmptyArray) + } + } +} + pub fn map( input: &RadonArray, args: &[Value], diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 32d6961c1..b1889c241 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -29,7 +29,7 @@ pub enum RadonOpCodes { // Array operator codes (start at 0x10) ArrayCount = 0x10, ArrayFilter = 0x11, - // ArrayFlatten = 0x12, + ArrayJoin = 0x12, ArrayGetArray = 0x13, ArrayGetBoolean = 0x14, ArrayGetBytes = 0x15, diff --git a/rad/src/types/array.rs b/rad/src/types/array.rs index 1c769e308..ce888f7f0 100644 --- a/rad/src/types/array.rs +++ b/rad/src/types/array.rs @@ -166,6 +166,7 @@ impl Operable for RadonArray { array_operators::get::(self, args).map(RadonTypes::from) } (RadonOpCodes::ArrayFilter, Some(args)) => array_operators::filter(self, args, context), + (RadonOpCodes::ArrayJoin, Some(args)) => array_operators::join(self, args), (RadonOpCodes::ArrayMap, Some(args)) => array_operators::map(self, args, context), (RadonOpCodes::ArrayReduce, Some(args)) => array_operators::reduce(self, args, context), (RadonOpCodes::ArraySort, Some(args)) => array_operators::sort(self, args, context), From 0ab36ce792791629905998f5bcf67737d7dbbde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 13 Dec 2023 11:06:51 +0100 Subject: [PATCH 21/46] chore(rad): rename asX <-> toX depending on whether conversion may fail --- rad/src/operators/mod.rs | 23 +++--- rad/src/operators/string.rs | 152 ++++++++++++++++++------------------ rad/src/types/boolean.rs | 2 +- rad/src/types/bytes.rs | 8 ++ rad/src/types/float.rs | 15 ++-- rad/src/types/integer.rs | 12 +-- rad/src/types/string.rs | 3 + 7 files changed, 112 insertions(+), 103 deletions(-) diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index b1889c241..00687a90c 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -44,41 +44,40 @@ pub enum RadonOpCodes { // ArrayTake = 0x1E, /////////////////////////////////////////////////////////////////////// // Boolean operator codes (start at 0x20) - BooleanAsString = 0x20, // BooleanMatch = 0x21, BooleanNegate = 0x22, + BooleanToString = 0x20, /////////////////////////////////////////////////////////////////////// - // Bytes operator codes (start at 0x30) - BytesAsString = 0x30, + // Bytes operator codes (start at 0x30) BytesHash = 0x31, + BytesLength = 0x34, + BytesSlice = 0x3C, + BytesStringify = 0x30, /////////////////////////////////////////////////////////////////////// // Integer operator codes (start at 0x40) IntegerAbsolute = 0x40, - IntegerAsFloat = 0x41, - IntegerAsString = 0x42, IntegerGreaterThan = 0x43, IntegerLessThan = 0x44, - // IntegerMatch = 0x45, IntegerModulo = 0x46, IntegerMultiply = 0x47, IntegerNegate = 0x48, IntegerPower = 0x49, - // IntegerReciprocal = 0x4A, + IntegerToFloat = 0x41, + IntegerToString = 0x42, // IntegerSum = 0x4B, /////////////////////////////////////////////////////////////////////// // Float operator codes (start at 0x50) FloatAbsolute = 0x50, - FloatAsString = 0x51, FloatCeiling = 0x52, - FloatGreaterThan = 0x53, FloatFloor = 0x54, + FloatGreaterThan = 0x53, FloatLessThan = 0x55, FloatModulo = 0x56, FloatMultiply = 0x57, FloatNegate = 0x58, FloatPower = 0x59, - // FloatReciprocal = 0x5A, FloatRound = 0x5B, + FloatToString = 0x51, // FloatSum = 0x5C, FloatTruncate = 0x5D, /////////////////////////////////////////////////////////////////////// @@ -104,11 +103,11 @@ pub enum RadonOpCodes { StringParseJSONArray = 0x76, StringParseJSONMap = 0x77, StringParseXMLMap = 0x78, - StringToLowerCase = 0x79, - StringToUpperCase = 0x7A, StringReplace = 0x7B, StringSlice = 0x7C, StringSplit = 0x7D, + StringToLowerCase = 0x79, + StringToUpperCase = 0x7A, } impl fmt::Display for RadonOpCodes { diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index 3149a9cc7..e27bfe48c 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -23,6 +23,71 @@ const MAX_DEPTH: u8 = 20; const DEFAULT_THOUSANDS_SEPARATOR: &str = ","; const DEFAULT_DECIMAL_SEPARATOR: &str = "."; +pub fn as_bool(input: &RadonString) -> Result { + let str_value = radon_trim(input); + bool::from_str(&str_value) + .map(RadonBoolean::from) + .map_err(Into::into) +} +/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents +/// a valid floating point number. +pub fn as_float(input: &RadonString, args: &Option>) -> Result { + f64::from_str(&as_numeric_string( + input, + args.as_deref().unwrap_or_default(), + )) + .map(RadonFloat::from) + .map_err(Into::into) +} + +/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents +/// a valid integer number. +pub fn as_integer( + input: &RadonString, + args: &Option>, +) -> Result { + i128::from_str(&as_numeric_string( + input, + args.as_deref().unwrap_or_default(), + )) + .map(RadonInteger::from) + .map_err(Into::into) +} + +/// Converts a `RadonString` into a `String` containing a numeric value, provided that the input +/// string actually represents a valid number. +pub fn as_numeric_string(input: &RadonString, args: &[Value]) -> String { + let str_value = radon_trim(input); + let (thousands_separator, decimal_separator) = read_separators_from_args(args); + + replace_separators(str_value, thousands_separator, decimal_separator) +} + +pub fn hash(input: &RadonString, args: &[Value]) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "Hash".to_string(), + args: args.to_vec(), + }; + + let input_string = input.value(); + let input_bytes = input_string.as_bytes(); + + let arg = args.first().ok_or_else(wrong_args)?.to_owned(); + let hash_function_integer = from_value::(arg).map_err(|_| wrong_args())?; + let hash_function_code = + RadonHashFunctions::try_from(hash_function_integer).map_err(|_| wrong_args())?; + + let digest = hash_functions::hash(input_bytes, hash_function_code)?; + let hex_string = hex::encode(digest); + + Ok(RadonString::from(hex_string)) +} + +pub fn length(input: &RadonString) -> RadonInteger { + RadonInteger::from(input.value().len() as i128) +} + /// Parse `RadonTypes` from a JSON-encoded `RadonString`. pub fn parse_json(input: &RadonString) -> Result { let json_value: JsonValue = @@ -153,81 +218,8 @@ pub fn radon_trim(input: &RadonString) -> String { } } -pub fn to_bool(input: &RadonString) -> Result { - let str_value = radon_trim(input); - bool::from_str(&str_value) - .map(RadonBoolean::from) - .map_err(Into::into) -} - -/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents -/// a valid floating point number. -pub fn as_float(input: &RadonString, args: &Option>) -> Result { - f64::from_str(&as_numeric_string( - input, - args.as_deref().unwrap_or_default(), - )) - .map(RadonFloat::from) - .map_err(Into::into) -} - -/// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents -/// a valid integer number. -pub fn as_integer( - input: &RadonString, - args: &Option>, -) -> Result { - i128::from_str(&as_numeric_string( - input, - args.as_deref().unwrap_or_default(), - )) - .map(RadonInteger::from) - .map_err(Into::into) -} - -/// Converts a `RadonString` into a `String` containing a numeric value, provided that the input -/// string actually represents a valid number. -pub fn as_numeric_string(input: &RadonString, args: &[Value]) -> String { - let str_value = radon_trim(input); - let (thousands_separator, decimal_separator) = read_separators_from_args(args); - - replace_separators(str_value, thousands_separator, decimal_separator) -} - -pub fn length(input: &RadonString) -> RadonInteger { - RadonInteger::from(input.value().len() as i128) -} - -pub fn to_lowercase(input: &RadonString) -> RadonString { - RadonString::from(input.value().as_str().to_lowercase()) -} - -pub fn to_uppercase(input: &RadonString) -> RadonString { - RadonString::from(input.value().as_str().to_uppercase()) -} - -pub fn hash(input: &RadonString, args: &[Value]) -> Result { - let wrong_args = || RadError::WrongArguments { - input_type: RadonString::radon_type_name(), - operator: "Hash".to_string(), - args: args.to_vec(), - }; - - let input_string = input.value(); - let input_bytes = input_string.as_bytes(); - - let arg = args.first().ok_or_else(wrong_args)?.to_owned(); - let hash_function_integer = from_value::(arg).map_err(|_| wrong_args())?; - let hash_function_code = - RadonHashFunctions::try_from(hash_function_integer).map_err(|_| wrong_args())?; - - let digest = hash_functions::hash(input_bytes, hash_function_code)?; - let hex_string = hex::encode(digest); - - Ok(RadonString::from(hex_string)) -} -pub fn string_replace(input: &RadonString, args: &[Value]) -> Result { +pub fn replace(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "StringReplace".to_string(), @@ -238,7 +230,7 @@ pub fn string_replace(input: &RadonString, args: &[Value]) -> Result Result { +pub fn slice(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "StringSlice".to_string(), @@ -259,7 +251,7 @@ pub fn string_slice(input: &RadonString, args: &[Value]) -> Result Result { +pub fn split(input: &RadonString, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "StringSplit".to_string(), @@ -327,6 +319,14 @@ pub fn string_match(input: &RadonString, args: &[Value]) -> Result RadonString { + RadonString::from(input.value().as_str().to_lowercase()) +} + +pub fn to_uppercase(input: &RadonString) -> RadonString { + RadonString::from(input.value().as_str().to_uppercase()) +} + /// Replace thousands and decimals separators in a `String`. #[inline] pub fn replace_separators( diff --git a/rad/src/types/boolean.rs b/rad/src/types/boolean.rs index f6af1fc64..3f64a3c34 100644 --- a/rad/src/types/boolean.rs +++ b/rad/src/types/boolean.rs @@ -80,7 +80,7 @@ impl Operable for RadonBoolean { match call { (RadonOpCodes::Identity, None) => identity(RadonTypes::from(self.clone())), (RadonOpCodes::BooleanNegate, None) => Ok(boolean_operators::negate(self).into()), - (RadonOpCodes::BooleanAsString, None) => boolean_operators::to_string(self.clone()) + (RadonOpCodes::BooleanToString, None) => boolean_operators::to_string(self.clone()) .map(RadonTypes::from) .map_err(Into::into), (op_code, args) => Err(RadError::UnsupportedOperator { diff --git a/rad/src/types/bytes.rs b/rad/src/types/bytes.rs index fe5c19198..8914e5982 100644 --- a/rad/src/types/bytes.rs +++ b/rad/src/types/bytes.rs @@ -90,6 +90,14 @@ impl Operable for RadonBytes { (RadonOpCodes::BytesHash, Some(args)) => bytes_operators::hash(self, args.as_slice()) .map(RadonTypes::from) .map_err(Into::into), + (RadonOpCodes::BytesLength, None) => { + Ok(RadonTypes::from(bytes_operators::length(self))) + } + (RadonOpCodes::BytesSlice, Some(args)) => { + bytes_operators::slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + }, + .map(RadonTypes::from) + .map_err(Into::into), // Unsupported / unimplemented (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_BYTES_TYPE_NAME.to_string(), diff --git a/rad/src/types/float.rs b/rad/src/types/float.rs index 3e81fb5d6..736c3d7f6 100644 --- a/rad/src/types/float.rs +++ b/rad/src/types/float.rs @@ -116,26 +116,22 @@ impl Operable for RadonFloat { (RadonOpCodes::FloatAbsolute, None) => { Ok(RadonTypes::from(float_operators::absolute(self))) } - (RadonOpCodes::FloatAsString, None) => float_operators::to_string(self.clone()) - .map(RadonTypes::from) - .map_err(Into::into), (RadonOpCodes::FloatCeiling, None) => { Ok(RadonTypes::from(float_operators::ceiling(self))) } + (RadonOpCodes::FloatFloor, None) => Ok(RadonTypes::from(float_operators::floor(self))), (RadonOpCodes::FloatGreaterThan, Some(args)) => { float_operators::greater_than(self, args).map(Into::into) } (RadonOpCodes::FloatLessThan, Some(args)) => { float_operators::less_than(self, args).map(Into::into) } - (RadonOpCodes::FloatMultiply, Some(args)) => { - float_operators::multiply(self, args.as_slice()).map(Into::into) - } (RadonOpCodes::FloatModulo, Some(args)) => { float_operators::modulo(self, args.as_slice()).map(Into::into) } - (RadonOpCodes::FloatFloor, None) => Ok(RadonTypes::from(float_operators::floor(self))), - + (RadonOpCodes::FloatMultiply, Some(args)) => { + float_operators::multiply(self, args.as_slice()).map(Into::into) + } (RadonOpCodes::FloatNegate, None) => { Ok(RadonTypes::from(float_operators::negate(self))) } @@ -143,6 +139,9 @@ impl Operable for RadonFloat { float_operators::power(self, args.as_slice()).map(Into::into) } (RadonOpCodes::FloatRound, None) => Ok(RadonTypes::from(float_operators::round(self))), + (RadonOpCodes::FloatToString, None) => float_operators::to_string(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::FloatTruncate, None) => { Ok(RadonTypes::from(float_operators::truncate(self))) } diff --git a/rad/src/types/integer.rs b/rad/src/types/integer.rs index 845fd7478..ed73c7217 100644 --- a/rad/src/types/integer.rs +++ b/rad/src/types/integer.rs @@ -105,12 +105,6 @@ impl Operable for RadonInteger { (RadonOpCodes::IntegerAbsolute, None) => integer_operators::absolute(self) .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::IntegerAsFloat, None) => integer_operators::to_float(self.clone()) - .map(RadonTypes::from) - .map_err(Into::into), - (RadonOpCodes::IntegerAsString, None) => integer_operators::to_string(self.clone()) - .map(RadonTypes::from) - .map_err(Into::into), (RadonOpCodes::IntegerGreaterThan, Some(args)) => { integer_operators::greater_than(self, args).map(Into::into) } @@ -129,6 +123,12 @@ impl Operable for RadonInteger { (RadonOpCodes::IntegerPower, Some(args)) => { integer_operators::power(self, args.as_slice()).map(Into::into) } + (RadonOpCodes::IntegerToFloat, None) => integer_operators::to_float(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), + (RadonOpCodes::IntegerToString, None) => integer_operators::to_string(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), // Unsupported / unimplemented (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_INTEGER_TYPE_NAME.to_string(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index d12fa79bd..cfdc382d3 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -105,6 +105,9 @@ impl Operable for RadonString { match call { (RadonOpCodes::Identity, None) => identity(RadonTypes::from(self.clone())), + (RadonOpCodes::StringAsBoolean, None) => string_operators::as_bool(self) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::StringAsFloat, args) => if wip0024 { string_operators::as_float(self, args) } else { From 1c7a4a2455840fb5c21e74cace34901504c12408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 13 Dec 2023 11:13:51 +0100 Subject: [PATCH 22/46] feat(rad): implement RadonOpCodes::BytesAsInteger --- rad/src/operators/bytes.rs | 37 +++++++++++++++++++++++++++++++++++-- rad/src/operators/mod.rs | 1 + rad/src/types/bytes.rs | 2 +- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/rad/src/operators/bytes.rs b/rad/src/operators/bytes.rs index 174e38b4d..363d994c2 100644 --- a/rad/src/operators/bytes.rs +++ b/rad/src/operators/bytes.rs @@ -7,8 +7,17 @@ use crate::{ types::{bytes::RadonBytes, string::RadonString, RadonType}, }; -pub fn to_string(input: &RadonBytes) -> Result { - RadonString::try_from(Value::Text(hex::encode(input.value()))) +pub fn as_integer(input: &RadonBytes) -> Result { + let input_value_len = input.value().len(); + match input_value_len { + 1..=16 => { + let mut bytes_array = [0u8; 16]; + bytes_array[16 - input_value_len ..].copy_from_slice(&input.value()); + Ok(RadonInteger::from(i128::from_be_bytes(bytes_array))) + } + 17.. => Err(RadError::ParseInt { message: "Input buffer too big".to_string() }), + _ => Err(RadError::EmptyArray) + } } pub fn hash(input: &RadonBytes, args: &[Value]) -> Result { @@ -27,6 +36,30 @@ pub fn hash(input: &RadonBytes, args: &[Value]) -> Result Ok(RadonBytes::from(digest)) } + +pub fn length(input: &RadonBytes) -> RadonInteger { + RadonInteger::from(input.value().len() as i128) +} + +pub fn slice(input: &RadonBytes, args: &[Value]) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "BytesSlice".to_string(), + args: args.to_vec(), + }; + let end_index = input.value().len(); + if end_index > 0 { + let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + let mut slice = input.value().as_slice().split_at(start_index).1.to_vec(); + if args.len() == 2 { + let end_index = from_value::(args[1].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + slice.truncate(end_index - start_index); + } + Ok(RadonBytes::from(slice)) + } else { + Err(wrong_args()) + } +} #[cfg(test)] mod tests { use super::*; diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 00687a90c..5470a14c9 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -49,6 +49,7 @@ pub enum RadonOpCodes { BooleanToString = 0x20, /////////////////////////////////////////////////////////////////////// // Bytes operator codes (start at 0x30) + BytesAsInteger = 0x32, BytesHash = 0x31, BytesLength = 0x34, BytesSlice = 0x3C, diff --git a/rad/src/types/bytes.rs b/rad/src/types/bytes.rs index 8914e5982..fd644ea85 100644 --- a/rad/src/types/bytes.rs +++ b/rad/src/types/bytes.rs @@ -84,7 +84,7 @@ impl Operable for RadonBytes { match call { // Identity (RadonOpCodes::Identity, None) => identity(RadonTypes::from(self.clone())), - (RadonOpCodes::BytesAsString, None) => bytes_operators::to_string(self) + (RadonOpCodes::BytesAsInteger, None) => bytes_operators::as_integer(self) .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::BytesHash, Some(args)) => bytes_operators::hash(self, args.as_slice()) From ae50ed87f8df78545f645c1f4a0b9df81be82e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 13 Dec 2023 11:16:09 +0100 Subject: [PATCH 23/46] feat(rad): support multiple encoding schemas on RadonOpCodes::BytesStringify --- rad/src/operators/bytes.rs | 37 ++++++++++++++++++++++++++++++++++--- rad/src/types/bytes.rs | 12 ++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/rad/src/operators/bytes.rs b/rad/src/operators/bytes.rs index 363d994c2..0106eac5c 100644 --- a/rad/src/operators/bytes.rs +++ b/rad/src/operators/bytes.rs @@ -1,10 +1,11 @@ +use base64::Engine; use serde_cbor::value::{from_value, Value}; use std::convert::TryFrom; use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, - types::{bytes::RadonBytes, string::RadonString, RadonType}, + types::{bytes::{RadonBytes, RadonBytesEncoding}, string::RadonString, integer::RadonInteger, RadonType}, }; pub fn as_integer(input: &RadonBytes) -> Result { @@ -60,14 +61,44 @@ pub fn slice(input: &RadonBytes, args: &[Value]) -> Result Err(wrong_args()) } } + +pub fn stringify(input: &RadonBytes, args: &Option>) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "Stringify".to_string(), + args: args.to_owned().unwrap_or(Vec::::default()).to_vec(), + }; + let mut bytes_encoding = RadonBytesEncoding::Hex; + match args { + Some(args) => { + if args.len() > 0 { + let arg = args.first().ok_or_else(wrong_args)?.to_owned(); + let bytes_encoding_u8 = from_value::(arg).map_err(|_| wrong_args())?; + bytes_encoding = RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; + } + } + _ => () + } + match bytes_encoding { + RadonBytesEncoding::Hex => { + RadonString::try_from(Value::Text(hex::encode(input.value()))) + } + RadonBytesEncoding::Base64 => { + RadonString::try_from(Value::Text(base64::engine::general_purpose::STANDARD.encode(input.value()))) + } + } + +} + #[cfg(test)] mod tests { use super::*; #[test] - fn test_bytes_to_string() { + fn test_bytes_stringify() { let input = RadonBytes::from(vec![0x01, 0x02, 0x03]); - let output = to_string(&input).unwrap().value(); + let valid_args = Some(vec![Value::from(0x00)]); + let output = stringify(&input, &valid_args).unwrap().value(); let valid_expected = "010203".to_string(); diff --git a/rad/src/types/bytes.rs b/rad/src/types/bytes.rs index fd644ea85..6f9637f01 100644 --- a/rad/src/types/bytes.rs +++ b/rad/src/types/bytes.rs @@ -4,6 +4,8 @@ use crate::{ script::RadonCall, types::{RadonType, RadonTypes}, }; +use num_enum::TryFromPrimitive; +use serde::Serialize; use serde_cbor::value::Value; use std::{ convert::{TryFrom, TryInto}, @@ -13,6 +15,15 @@ use witnet_data_structures::radon_report::ReportContext; const RADON_BYTES_TYPE_NAME: &str = "RadonBytes"; +/// List of support string-encoding algorithms for buffers +#[derive(Debug, Default, PartialEq, Eq, Serialize, TryFromPrimitive)] +#[repr(u8)] +pub enum RadonBytesEncoding { + #[default] + Hex = 0, + Base64 = 1, +} + #[derive(Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] pub struct RadonBytes { value: Vec, @@ -96,6 +107,7 @@ impl Operable for RadonBytes { (RadonOpCodes::BytesSlice, Some(args)) => { bytes_operators::slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) }, + (RadonOpCodes::BytesStringify, args) => bytes_operators::stringify(self, args) .map(RadonTypes::from) .map_err(Into::into), // Unsupported / unimplemented From 888ef3ba656d9908c33ec3fa8b429678e9b8449d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 13 Dec 2023 11:17:27 +0100 Subject: [PATCH 24/46] feat(rad): implement RadonOpCodes::IntegerToBytes --- rad/src/operators/integer.rs | 16 +++++++++++++++- rad/src/operators/mod.rs | 1 + rad/src/types/integer.rs | 3 +++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/rad/src/operators/integer.rs b/rad/src/operators/integer.rs index 407522dfa..49b3a4b00 100644 --- a/rad/src/operators/integer.rs +++ b/rad/src/operators/integer.rs @@ -6,7 +6,7 @@ use crate::{ error::RadError, types::{ boolean::RadonBoolean, float::RadonFloat, integer::RadonInteger, string::RadonString, - RadonType, + RadonType, bytes::RadonBytes, }, }; @@ -20,6 +20,20 @@ pub fn absolute(input: &RadonInteger) -> Result { } } +pub fn to_bytes(input: RadonInteger) -> Result { + let mut bytes_array = [0u8; 16]; + bytes_array.copy_from_slice(&input.value().to_be_bytes()); + let mut leading_zeros = 0; + for i in 0..bytes_array.len() { + if bytes_array[i] != 0u8 { + break + } else { + leading_zeros += 1; + } + } + Ok(RadonBytes::from(bytes_array[leading_zeros..].to_vec())) +} + pub fn to_float(input: RadonInteger) -> Result { RadonFloat::try_from(Value::Integer(input.value())) } diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 5470a14c9..1163a8df0 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -63,6 +63,7 @@ pub enum RadonOpCodes { IntegerMultiply = 0x47, IntegerNegate = 0x48, IntegerPower = 0x49, + IntegerToBytes = 0x4A, IntegerToFloat = 0x41, IntegerToString = 0x42, // IntegerSum = 0x4B, diff --git a/rad/src/types/integer.rs b/rad/src/types/integer.rs index ed73c7217..254ff70d9 100644 --- a/rad/src/types/integer.rs +++ b/rad/src/types/integer.rs @@ -123,6 +123,9 @@ impl Operable for RadonInteger { (RadonOpCodes::IntegerPower, Some(args)) => { integer_operators::power(self, args.as_slice()).map(Into::into) } + (RadonOpCodes::IntegerToBytes, None) => integer_operators::to_bytes(self.clone()) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::IntegerToFloat, None) => integer_operators::to_float(self.clone()) .map(RadonTypes::from) .map_err(Into::into), From 179503c78fca80b78cd304584f0accd5cd5b8c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 13 Dec 2023 11:19:39 +0100 Subject: [PATCH 25/46] feat(rad): implement RadonOpCodes::StringAsBytes --- Cargo.lock | 9 ++++--- rad/Cargo.toml | 1 + rad/src/operators/mod.rs | 2 +- rad/src/operators/string.rs | 52 +++++++++++++++++++++++++++++++++++-- rad/src/types/string.rs | 30 ++++++++++----------- 5 files changed, 72 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de24156e0..4e928143a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -242,9 +242,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bech32" @@ -3252,7 +3252,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.2", + "base64 0.21.5", "bytes 1.4.0", "encoding_rs", "futures-core", @@ -4496,7 +4496,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" dependencies = [ - "base64 0.21.2", + "base64 0.21.5", "log 0.4.19", "native-tls", "once_cell", @@ -5206,6 +5206,7 @@ dependencies = [ name = "witnet_rad" version = "0.3.3" dependencies = [ + "base64 0.21.5", "cbor-codec", "failure", "futures 0.3.28", diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 2a0d478de..09c73d78a 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -10,6 +10,7 @@ description = "RAD component" rocksdb-backend = ["witnet_data_structures/rocksdb-backend"] [dependencies] +base64 = "0.21.5" cbor-codec = { git = "https://github.com/witnet/cbor-codec.git", branch = "feat/ldexpf-shim" } failure = "0.1.8" futures = "0.3.4" diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 1163a8df0..b104c655c 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -97,7 +97,7 @@ pub enum RadonOpCodes { /////////////////////////////////////////////////////////////////////// // String operator codes (start at 0x70) StringAsBoolean = 0x70, - // StringAsBytes = 0x71, + StringAsBytes = 0x71, StringAsFloat = 0x72, StringAsInteger = 0x73, StringLength = 0x74, diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index e27bfe48c..df6fc7a8f 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -4,6 +4,7 @@ use std::{ str::FromStr, }; +use base64::Engine; use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; @@ -14,7 +15,7 @@ use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, types::{ - array::RadonArray, boolean::RadonBoolean, bytes::RadonBytes, float::RadonFloat, + array::RadonArray, boolean::RadonBoolean, bytes::{RadonBytes, RadonBytesEncoding}, float::RadonFloat, integer::RadonInteger, map::RadonMap, string::RadonString, RadonType, RadonTypes, }, }; @@ -29,6 +30,53 @@ pub fn as_bool(input: &RadonString) -> Result { .map(RadonBoolean::from) .map_err(Into::into) } + +pub fn as_bytes(input: &RadonString, args: &Option>) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "AsBytes".to_string(), + args: args.to_owned().unwrap_or(Vec::::default()).to_vec(), + }; + let mut input_string = input.value(); + if input_string.starts_with("0x") { + input_string = input_string.slice(2..); + } + if input_string.len() % 2 != 0 { + input_string.insert(0, '0'); + } + let mut bytes_encoding = RadonBytesEncoding::Hex; + match args { + Some(args) => { + if args.len() > 0 { + let arg = args.first().ok_or_else(wrong_args)?.to_owned(); + let bytes_encoding_u8 = from_value::(arg).map_err(|_| wrong_args())?; + bytes_encoding = RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; + } + } + _ => () + } + match bytes_encoding { + RadonBytesEncoding::Hex => { + Ok(RadonBytes::from( + hex::decode(input_string.as_str()) + .map_err(|_err| RadError::Decode { + from: "RadonString", + to: "RadonBytes", + })? + )) + } + RadonBytesEncoding::Base64 => { + Ok(RadonBytes::from( + base64::engine::general_purpose::STANDARD.decode(input.value()) + .map_err(|_err| RadError::Decode { + from: "RadonString", + to: "RadonBytes" + })? + )) + } + } +} + /// Converts a `RadonString` into a `RadonFloat`, provided that the input string actually represents /// a valid floating point number. pub fn as_float(input: &RadonString, args: &Option>) -> Result { @@ -777,7 +825,7 @@ mod tests { let rad_float = RadonBoolean::from(false); let rad_string: RadonString = RadonString::from("false"); - assert_eq!(to_bool(&rad_string).unwrap(), rad_float); + assert_eq!(as_bool(&rad_string).unwrap(), rad_float); } #[test] diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index cfdc382d3..21ca21111 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -108,6 +108,9 @@ impl Operable for RadonString { (RadonOpCodes::StringAsBoolean, None) => string_operators::as_bool(self) .map(RadonTypes::from) .map_err(Into::into), + (RadonOpCodes::StringAsBytes, args) => string_operators::as_bytes(self, args) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::StringAsFloat, args) => if wip0024 { string_operators::as_float(self, args) } else { @@ -122,38 +125,35 @@ impl Operable for RadonString { } .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::StringAsBoolean, None) => string_operators::to_bool(self) - .map(RadonTypes::from) - .map_err(Into::into), + (RadonOpCodes::StringLength, None) => { + Ok(RadonTypes::from(string_operators::length(self))) + } + (RadonOpCodes::StringMatch, Some(args)) => { + string_operators::string_match(self, args.as_slice()).map(RadonTypes::from) + } (RadonOpCodes::StringParseJSONArray, None) => string_operators::parse_json_array(self) .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::StringParseJSONMap, None) => string_operators::parse_json_map(self) .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::StringMatch, Some(args)) => { - string_operators::string_match(self, args.as_slice()).map(RadonTypes::from) - } - (RadonOpCodes::StringLength, None) => { - Ok(RadonTypes::from(string_operators::length(self))) - } + (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::StringToLowerCase, None) => { Ok(RadonTypes::from(string_operators::to_lowercase(self))) } (RadonOpCodes::StringToUpperCase, None) => { Ok(RadonTypes::from(string_operators::to_uppercase(self))) } - (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) - .map(RadonTypes::from) - .map_err(Into::into), (RadonOpCodes::StringReplace, Some(args)) => { - string_operators::string_replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + string_operators::replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } (RadonOpCodes::StringSlice, Some(args)) => { - string_operators::string_slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + string_operators::slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } (RadonOpCodes::StringSplit, Some(args)) => { - string_operators::string_split(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + string_operators::split(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), From 6802cad9c3fd77a94cc24100b167e3d3d6c12404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 13 Dec 2023 18:02:15 +0100 Subject: [PATCH 26/46] feat(rad): include retrieval response time within WitnetResultReports --- rad/src/lib.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 289178f98..4507a803e 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -276,6 +276,7 @@ async fn http_response( }) })?; + let start_ts = std::time::SystemTime::now(); let response = client .send(request) .await @@ -322,7 +323,19 @@ async fn http_response( response = RadonTypes::from(RadonString::from(response_string)); } - let result = handle_response_with_data_report(retrieve, response, context, settings); + let result = handle_response_with_data_report(retrieve, response, context, settings) + .map(|report| { + let completion_ts = std::time::SystemTime::now(); + RadonReport { + context: ReportContext { + start_time: Some(start_ts), + completion_time: Some(completion_ts), + ..report.context + }, + running_time: completion_ts.duration_since(start_ts).unwrap_or_default(), + ..report + } + }); match &result { Ok(report) => { log::debug!( From 73d49e7b433272841f820c1e389afacf668c11f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 09:18:22 +0100 Subject: [PATCH 27/46] feat: implement RadonOpCodes::ArrayPick --- rad/src/operators/array.rs | 46 ++++++++++++++++++++++++++++++++++++++ rad/src/operators/mod.rs | 2 +- rad/src/types/array.rs | 1 + 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index c86f7778e..53f7a4fd7 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -255,6 +255,52 @@ pub fn filter( } } +pub fn pick( + input: &RadonArray, + args: &[Value], + _context: &mut ReportContext +) -> Result { + let not_found = |index: usize| RadError::ArrayIndexOutOfBounds { + index: i32::try_from(index).unwrap(), + }; + + let wrong_args = || RadError::WrongArguments { + input_type: RadonArray::radon_type_name(), + operator: "Pick".to_string(), + args: args.to_vec(), + }; + + let mut indexes = vec![]; + if args.len() > 1 { + return Err(wrong_args()); + } else { + let first_arg = args.get(0).ok_or_else(wrong_args)?; + match first_arg { + Value::Array(values) => { + for value in values.iter() { + let index = from_value::(value.clone()).map_err(|_| wrong_args())?; + indexes.push(index); + } + } + Value::Integer(_) => { + let index = from_value::(first_arg.clone()).map_err(|_| wrong_args())?; + indexes.push(index); + } + _ => return Err(wrong_args()) + }; + } + + let mut output_vec: Vec = vec![]; + for index in indexes { + if let Some(value) = input.value().get(index) { + output_vec.push(value.clone()); + } else { + return Err(not_found(index)); + } + } + Ok(RadonTypes::from(RadonArray::from(output_vec))) +} + pub fn sort( input: &RadonArray, args: &[Value], diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index b104c655c..f8a0d4980 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -41,7 +41,7 @@ pub enum RadonOpCodes { ArrayReduce = 0x1B, // ArraySome = 0x1C, ArraySort = 0x1D, - // ArrayTake = 0x1E, + ArrayPick = 0x1E, /////////////////////////////////////////////////////////////////////// // Boolean operator codes (start at 0x20) // BooleanMatch = 0x21, diff --git a/rad/src/types/array.rs b/rad/src/types/array.rs index ce888f7f0..744f7a827 100644 --- a/rad/src/types/array.rs +++ b/rad/src/types/array.rs @@ -170,6 +170,7 @@ impl Operable for RadonArray { (RadonOpCodes::ArrayMap, Some(args)) => array_operators::map(self, args, context), (RadonOpCodes::ArrayReduce, Some(args)) => array_operators::reduce(self, args, context), (RadonOpCodes::ArraySort, Some(args)) => array_operators::sort(self, args, context), + (RadonOpCodes::ArrayPick, Some(args)) => array_operators::pick(self, args, context), (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_ARRAY_TYPE_NAME.to_string(), operator: op_code.to_string(), From 3ce8d2d079b1ee75e4c130e5b34e672c7336112e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 09:22:44 +0100 Subject: [PATCH 28/46] feat: implement RadonOpCodes::MapStringify --- rad/src/operators/map.rs | 8 ++++++++ rad/src/operators/mod.rs | 2 +- rad/src/types/map.rs | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/rad/src/operators/map.rs b/rad/src/operators/map.rs index c81da9eef..b2964d38f 100644 --- a/rad/src/operators/map.rs +++ b/rad/src/operators/map.rs @@ -80,6 +80,14 @@ pub fn values(input: &RadonMap) -> RadonArray { let v: Vec = input.value().values().cloned().collect(); RadonArray::from(v) } +pub fn stringify(input: &RadonMap) -> Result { + let json_string = serde_json::to_string(&input.value()) + .map_err(|_| RadError::Decode { + from: "RadonMap", + to: "RadonString" + })?; + Ok(RadonString::from(json_string)) +} /// This module was introduced for encapsulating the interim legacy logic before WIP-0024 is /// introduced, for the sake of maintainability. diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index f8a0d4980..23f21a6dd 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -84,7 +84,7 @@ pub enum RadonOpCodes { FloatTruncate = 0x5D, /////////////////////////////////////////////////////////////////////// // Map operator codes (start at 0x60) - // MapEntries = 0x60, + MapStringify = 0x60, MapGetArray = 0x61, MapGetBoolean = 0x62, MapGetBytes = 0x63, diff --git a/rad/src/types/map.rs b/rad/src/types/map.rs index 41e6d4380..7ab9e379b 100644 --- a/rad/src/types/map.rs +++ b/rad/src/types/map.rs @@ -161,6 +161,7 @@ impl Operable for RadonMap { } (RadonOpCodes::MapKeys, None) => Ok(RadonTypes::from(map_operators::keys(self))), (RadonOpCodes::MapValues, None) => Ok(RadonTypes::from(map_operators::values(self))), + (RadonOpCodes::MapStringify, None) => map_operators::stringify(self).map(RadonTypes::from), (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_MAP_TYPE_NAME.to_string(), operator: op_code.to_string(), From aae16d241a3b2171bbbdafac1b918d39649771c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 09:29:49 +0100 Subject: [PATCH 29/46] feat: implement RadonOpCodes::MapAlter --- rad/src/operators/array.rs | 34 +++-------------- rad/src/operators/map.rs | 78 +++++++++++++++++++++++++++++++++++++- rad/src/operators/mod.rs | 2 + rad/src/script.rs | 29 +++++++++++++- rad/src/types/map.rs | 1 + 5 files changed, 112 insertions(+), 32 deletions(-) diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index 53f7a4fd7..4d483dc65 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -1,18 +1,17 @@ use std::{ clone::Clone, - convert::{TryFrom, TryInto}, - iter, + convert::{TryFrom, TryInto} }; use serde_cbor::value::{from_value, Value}; -use witnet_data_structures::radon_report::{RadonReport, ReportContext, Stage}; +use witnet_data_structures::radon_report::ReportContext; use crate::{ error::RadError, filters::{self, RadonFilters}, - operators::{string, RadonOpCodes}, + operators::string, reducers::{self, RadonReducers}, - script::{execute_radon_script, unpack_subscript, RadonCall, RadonScriptExecutionSettings}, + script::{execute_radon_script, unpack_subscript, RadonScriptExecutionSettings, partial_results_extract}, types::{array::RadonArray, integer::RadonInteger, string::RadonString, RadonType, RadonTypes}, }; @@ -379,27 +378,6 @@ pub fn sort( Ok(RadonArray::from(result).into()) } -fn partial_results_extract( - subscript: &[RadonCall], - reports: &[RadonReport], - context: &mut ReportContext, -) { - if let Stage::Retrieval(metadata) = &mut context.stage { - metadata.subscript_partial_results.push(subscript.iter().chain(iter::once(&(RadonOpCodes::Fail, None))).enumerate().map(|(index, _)| - reports - .iter() - .map(|report| - report.partial_results - .as_ref() - .expect("Execution reports from applying subscripts are expected to contain partial results") - .get(index) - .expect("Execution reports from applying same subscript on multiple values should contain the same number of partial results") - .clone() - ).collect::>() - ).collect::>>()); - } -} - pub fn transpose(input: &RadonArray) -> Result { let mut v = vec![]; let mut prev_len = None; @@ -486,7 +464,7 @@ pub mod legacy { mod tests { use std::collections::BTreeMap; - use witnet_data_structures::radon_report::RetrievalMetadata; + use witnet_data_structures::radon_report::{RetrievalMetadata, Stage}; use crate::{ error::RadError, @@ -494,7 +472,7 @@ mod tests { Operable, RadonOpCodes::{ IntegerGreaterThan, IntegerMultiply, MapGetBoolean, MapGetFloat, MapGetInteger, - MapGetString, + MapGetString, self, }, }, types::{ diff --git a/rad/src/operators/map.rs b/rad/src/operators/map.rs index b2964d38f..db2dfc752 100644 --- a/rad/src/operators/map.rs +++ b/rad/src/operators/map.rs @@ -1,13 +1,87 @@ -use std::convert::TryInto; +use std::{convert::TryInto, collections::BTreeMap}; use serde_cbor::value::{from_value, Value}; +use witnet_data_structures::radon_report::ReportContext; use crate::{ error::RadError, operators::string, - types::{array::RadonArray, map::RadonMap, string::RadonString, RadonType, RadonTypes}, + types::{array::RadonArray, map::RadonMap, string::RadonString, RadonType, RadonTypes}, + script::{RadonScriptExecutionSettings, execute_radon_script, unpack_subscript, partial_results_extract}, }; +pub fn alter( + input: &RadonMap, + args: &[Value], + context: &mut ReportContext +) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonMap::radon_type_name(), + operator: "Alter".to_string(), + args: args.to_vec(), + }; + + let first_arg = args.get(0).ok_or_else(wrong_args)?; + let mut input_keys = vec![]; + match first_arg { + Value::Array(keys) => { + for key in keys.iter() { + let key_string = from_value::(key.to_owned()).map_err(|_| wrong_args())?; + input_keys.push(key_string); + } + } + Value::Text(key) => { + input_keys.push(key.clone()); + } + _ => return Err(wrong_args()) + }; + + let subscript = args.get(1).ok_or_else(wrong_args)?; + match subscript { + Value::Array(_arg) => { + let subscript_err = |e| RadError::Subscript { + input_type: "RadonMap".to_string(), + operator: "Alter".to_string(), + inner: Box::new(e), + }; + let subscript = unpack_subscript(subscript).map_err(subscript_err)?; + + let not_found = |key_str: &str| RadError::MapKeyNotFound { key: String::from(key_str) }; + + let input_map = input.value(); + let mut output_map = input.value().clone(); + let mut reports = vec![]; + + let settings = RadonScriptExecutionSettings::tailored_to_stage(&context.stage); + for key in input_keys { + let value = input_map + .get(key.as_str()) + .ok_or_else(|| not_found(key.as_str()))?; + let report = execute_radon_script( + value.clone(), + subscript.as_slice(), + context, + settings + )?; + // If there is an error while altering value, short-circuit and bubble up the error as it comes + // from the radon script execution + if let RadonTypes::RadonError(error) = &report.result { + return Err(error.clone().into_inner()); + } else { + output_map.insert(key, report.result.clone()); + } + reports.push(report); + } + + // Extract the partial results from the reports and put them in the execution context if needed + partial_results_extract(&subscript, &reports, context); + + Ok(RadonMap::from(output_map)) + } + _ => Err(wrong_args()) + } +} + fn inner_get(input: &RadonMap, args: &[Value]) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonMap::radon_type_name(), diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 23f21a6dd..acd18f469 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -94,6 +94,8 @@ pub enum RadonOpCodes { MapGetString = 0x67, MapKeys = 0x68, MapValues = 0x69, + //MapEntries = 0x6A, + MapAlter = 0x6B, /////////////////////////////////////////////////////////////////////// // String operator codes (start at 0x70) StringAsBoolean = 0x70, diff --git a/rad/src/script.rs b/rad/src/script.rs index c395b757a..a203ef477 100644 --- a/rad/src/script.rs +++ b/rad/src/script.rs @@ -1,4 +1,8 @@ -use std::convert::TryFrom; +use std::{ + clone::Clone, + convert::TryFrom, + iter, +}; use serde_cbor::{ self as cbor, @@ -280,6 +284,27 @@ pub fn create_radon_script_from_filters_and_reducer( Ok(radoncall_vec) } +pub fn partial_results_extract( + subscript: &[RadonCall], + reports: &[RadonReport], + context: &mut ReportContext, +) { + if let Stage::Retrieval(metadata) = &mut context.stage { + metadata.subscript_partial_results.push(subscript.iter().chain(iter::once(&(RadonOpCodes::Fail, None))).enumerate().map(|(index, _)| + reports + .iter() + .map(|report| + report.partial_results + .as_ref() + .expect("Execution reports from applying subscripts are expected to contain partial results") + .get(index) + .expect("Execution reports from applying same subscript on multiple values should contain the same number of partial results") + .clone() + ).collect::>() + ).collect::>>()); + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -299,7 +324,7 @@ mod tests { r#"{"coord":{"lon":13.41,"lat":52.52},"weather":[{"id":600,"main":"Snow","description":"light snow","icon":"13n"}],"base":"stations","main":{"temp":-4,"pressure":1013,"humidity":73,"temp_min":-4,"temp_max":-4},"visibility":10000,"wind":{"speed":2.6,"deg":90},"clouds":{"all":75},"dt":1548346800,"sys":{"type":1,"id":1275,"message":0.0038,"country":"DE","sunrise":1548313160,"sunset":1548344298},"id":2950159,"name":"Berlin","cod":200}"#, )); let script = vec![ - (RadonOpCodes::StringParseJSONMap, None), + (RadonOpCodes::StringParseJSONMap, Some(vec![])), ( RadonOpCodes::MapGetMap, Some(vec![Value::Text(String::from("main"))]), diff --git a/rad/src/types/map.rs b/rad/src/types/map.rs index 7ab9e379b..c041bcbc7 100644 --- a/rad/src/types/map.rs +++ b/rad/src/types/map.rs @@ -161,6 +161,7 @@ impl Operable for RadonMap { } (RadonOpCodes::MapKeys, None) => Ok(RadonTypes::from(map_operators::keys(self))), (RadonOpCodes::MapValues, None) => Ok(RadonTypes::from(map_operators::values(self))), + (RadonOpCodes::MapAlter, Some(args)) => map_operators::alter(self, args, context).map(RadonTypes::from), (RadonOpCodes::MapStringify, None) => map_operators::stringify(self).map(RadonTypes::from), (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_MAP_TYPE_NAME.to_string(), From f4926768ee1da612ddfc10bde6d13346925edb0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 09:37:13 +0100 Subject: [PATCH 30/46] feat: implement RadonOpCodes::MapPick --- rad/src/operators/map.rs | 42 ++++++++++++++++++++++++++++++++++++++++ rad/src/operators/mod.rs | 1 + rad/src/types/map.rs | 1 + 3 files changed, 44 insertions(+) diff --git a/rad/src/operators/map.rs b/rad/src/operators/map.rs index db2dfc752..b4bd959bf 100644 --- a/rad/src/operators/map.rs +++ b/rad/src/operators/map.rs @@ -154,6 +154,48 @@ pub fn values(input: &RadonMap) -> RadonArray { let v: Vec = input.value().values().cloned().collect(); RadonArray::from(v) } + +pub fn pick(input: &RadonMap, args: &[Value]) -> Result { + let not_found = |key_str: &str| RadError::MapKeyNotFound { + key: String::from(key_str) + }; + + let wrong_args = || RadError::WrongArguments { + input_type: RadonMap::radon_type_name(), + operator: "Pick".to_string(), + args: args.to_vec(), + }; + + let mut input_keys = vec![]; + if args.len() > 1 { + return Err(wrong_args()); + } else { + let first_arg = args.get(0).ok_or_else(wrong_args)?; + match first_arg { + Value::Array(keys) => { + for key in keys.iter() { + let key_string = from_value::(key.to_owned()).map_err(|_| wrong_args())?; + input_keys.push(key_string); + } + } + Value::Text(key) => { + input_keys.push(key.clone()); + } + _ => return Err(wrong_args()) + }; + } + + let mut output_map = BTreeMap::::default(); + for key in input_keys { + if let Some(value) = input.value().get(&key) { + output_map.insert(key, value.clone()); + } else { + return Err(not_found(key.as_str())) + } + } + Ok(RadonMap::from(output_map)) +} + pub fn stringify(input: &RadonMap) -> Result { let json_string = serde_json::to_string(&input.value()) .map_err(|_| RadError::Decode { diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index acd18f469..9afdbb399 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -96,6 +96,7 @@ pub enum RadonOpCodes { MapValues = 0x69, //MapEntries = 0x6A, MapAlter = 0x6B, + MapPick = 0x6E, /////////////////////////////////////////////////////////////////////// // String operator codes (start at 0x70) StringAsBoolean = 0x70, diff --git a/rad/src/types/map.rs b/rad/src/types/map.rs index c041bcbc7..c86f294a1 100644 --- a/rad/src/types/map.rs +++ b/rad/src/types/map.rs @@ -162,6 +162,7 @@ impl Operable for RadonMap { (RadonOpCodes::MapKeys, None) => Ok(RadonTypes::from(map_operators::keys(self))), (RadonOpCodes::MapValues, None) => Ok(RadonTypes::from(map_operators::values(self))), (RadonOpCodes::MapAlter, Some(args)) => map_operators::alter(self, args, context).map(RadonTypes::from), + (RadonOpCodes::MapPick, Some(args)) => map_operators::pick(self, args).map(RadonTypes::from), (RadonOpCodes::MapStringify, None) => map_operators::stringify(self).map(RadonTypes::from), (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_MAP_TYPE_NAME.to_string(), From f222ee5a1484fd5f6402a5494ecc2ac3bb15d6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 09:42:33 +0100 Subject: [PATCH 31/46] feat: support optional json path as first argument on RadonOpCodes::StringParseJSONMap --- Cargo.lock | 74 ++++++++++++++++++++++++++++++++++++- rad/Cargo.toml | 1 + rad/src/error.rs | 9 +++++ rad/src/operators/string.rs | 42 +++++++++++++++++---- rad/src/types/string.rs | 2 +- 5 files changed, 118 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e928143a..090ca6901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,7 +147,7 @@ name = "async-jsonrpc-client" version = "0.1.0" source = "git+https://github.com/witnet/async-jsonrpc-client?branch=fix-tcp-leak#600a2d696af265511868f77c01e448c1d678650e" dependencies = [ - "error-chain", + "error-chain 0.12.4", "futures 0.1.31", "jsonrpc-core 10.1.0", "log 0.4.19", @@ -929,6 +929,15 @@ dependencies = [ "libc", ] +[[package]] +name = "error-chain" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" +dependencies = [ + "backtrace", +] + [[package]] name = "error-chain" version = "0.12.4" @@ -1753,6 +1762,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonpath" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8061db09019f1879ba586685694fe18279f597b1b3a9dd308f35e596be6cdf7d" +dependencies = [ + "error-chain 0.11.0", + "pest", + "pest_derive", + "serde", + "serde_json", +] + [[package]] name = "jsonrpc-core" version = "10.1.0" @@ -2744,6 +2766,23 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" + +[[package]] +name = "pest_derive" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3294f437119209b084c797604295f40227cffa35c57220b1e99a6ff3bf8ee4" +dependencies = [ + "pest", + "quote 0.3.15", + "syn 0.11.11", +] + [[package]] name = "pin-project" version = "1.1.0" @@ -2968,6 +3007,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" + [[package]] name = "quote" version = "0.6.13" @@ -3826,6 +3871,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +dependencies = [ + "quote 0.3.15", + "synom", + "unicode-xid 0.0.4", +] + [[package]] name = "syn" version = "0.15.44" @@ -3859,6 +3915,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +dependencies = [ + "unicode-xid 0.0.4", +] + [[package]] name = "synstructure" version = "0.12.6" @@ -4472,6 +4537,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -5213,6 +5284,7 @@ dependencies = [ "hex", "http", "if_rust_version", + "jsonpath", "log 0.4.19", "minidom", "num_enum", diff --git a/rad/Cargo.toml b/rad/Cargo.toml index 09c73d78a..4cd5b3080 100644 --- a/rad/Cargo.toml +++ b/rad/Cargo.toml @@ -19,6 +19,7 @@ if_rust_version = "1.0.0" # the http crate is used to perform additional validations before passing arguments to the surf http client # the version of http must be kept in sync with the version used by surf http = "0.2.1" +jsonpath = "0.1.1" log = "0.4.8" minidom = { git = "https://github.com/witnet/xmpp-rs", rev = "bc8a33ff5da95ee4039ad7ee3376c100d9e35c74" } num_enum = "0.4.2" diff --git a/rad/src/error.rs b/rad/src/error.rs index b9f2086b6..c2c606ac7 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -38,6 +38,15 @@ pub enum RadError { description )] JsonParse { description: String }, + /// The given JSON path is not present in a JSON-stringified object + #[fail(display = "Failed to find JSON path `{}` from RadonString", path)] + JsonPathNotFound { path: String }, + /// Failed to parse a JSON path selector from a string value + #[fail( + display = "Failed to parse a JSON path from a string value: {:?}", + description + )] + JsonPathParse { description: String }, /// Failed to parse an object from a XML buffer #[fail( display = "Failed to parse an object from a XML buffer: {:?}", diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index df6fc7a8f..c766eb88d 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -7,9 +7,11 @@ use std::{ use base64::Engine; use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; +use jsonpath::Selector; use slicestring::Slice; use regex::Regex; +use witnet_data_structures::radon_error::RadonError; use crate::{ error::RadError, @@ -142,18 +144,42 @@ pub fn parse_json(input: &RadonString) -> Result { serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse { description: err.to_string(), })?; - RadonTypes::try_from(json_value) } -pub fn parse_json_map(input: &RadonString) -> Result { - let item = parse_json(input)?; - item.try_into() -} +pub fn parse_json_map(input: &RadonString, args: &Option>) -> Result { + let not_found = |json_path: &str| RadError::JsonPathNotFound { + path: String::from(json_path) + }; + + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "ParseJsonMap".to_string(), + args: args.to_owned().unwrap_or_default(), + }; + + let json_input: JsonValue = serde_json::from_str(&input.value()) + .map_err(|err| RadError::JsonParse { + description: err.to_string(), + })?; + + match args.to_owned().unwrap_or_default().get(0) { + Some(Value::Text(json_path)) => { + let selector = Selector::new(json_path.as_str()) + .map_err(|err| RadError::JsonPathParse { + description: err.to_string(), + })?; + let item = selector.find(&json_input) + .next() + .ok_or_else(|| not_found(json_path.as_str()))?; + RadonTypes::try_from(item.to_owned())?.try_into() + }, + None => { + RadonTypes::try_from(json_input)?.try_into() + }, + _ => Err(wrong_args()) + } -pub fn parse_json_array(input: &RadonString) -> Result { - let item = parse_json(input)?; - item.try_into() } fn add_children( diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index 21ca21111..5c3bed08f 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -134,7 +134,7 @@ impl Operable for RadonString { (RadonOpCodes::StringParseJSONArray, None) => string_operators::parse_json_array(self) .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::StringParseJSONMap, None) => string_operators::parse_json_map(self) + (RadonOpCodes::StringParseJSONMap, args) => string_operators::parse_json_map(self, &args) .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) From 616958b6f8135cc1d4b384c286b3e9cd189bc5a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 09:47:12 +0100 Subject: [PATCH 32/46] feat: support optional json path as first argument on RadonOpCodes::StringParseJSONArray --- rad/src/operators/array.rs | 1 + rad/src/operators/string.rs | 98 +++++++++++++++++++++++++++++++++---- rad/src/types/string.rs | 2 +- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index 4d483dc65..94808287b 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -129,6 +129,7 @@ pub fn join( RadonString::try_from(item).unwrap_or_default().value() ).collect(); Ok(RadonTypes::from(RadonString::from(string_list.join(separator.as_str())))) + } Some(first_item) => { return Err(RadError::UnsupportedOperator { input_type: first_item.radon_type_name().to_string(), diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index c766eb88d..58d1faae8 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -179,7 +179,87 @@ pub fn parse_json_map(input: &RadonString, args: &Option>) -> Result< }, _ => Err(wrong_args()) } +} + +pub fn parse_json_array(input: &RadonString, args: &Option>) -> Result { + let wrong_args = || RadError::WrongArguments { + input_type: RadonString::radon_type_name(), + operator: "ParseJsonArray".to_string(), + args: args.to_owned().unwrap_or_default(), + }; + + let json_input: JsonValue = serde_json::from_str(&input.value()) + .map_err(|err| RadError::JsonParse { + description: err.to_string(), + })?; + + match args.to_owned().unwrap_or_default().get(0) { + Some(Value::Array(values)) => { + let mut items: Vec = vec![]; + for path in values { + if let Value::Text(json_path) = path { + let selector = Selector::new(json_path.as_str()) + .map_err(|err| RadError::JsonPathParse { + description: err.to_string(), + })?; + let mut subitems: Vec = selector.find(&json_input) + .map(|item| into_radon_types(item)) + .collect(); + if subitems.len() > 1 { + items.insert(items.len(), RadonArray::from(subitems).into()); + } else { + items.append(subitems.as_mut()); + } + } else { + return Err(wrong_args()); + } + } + Ok(RadonArray::from(items)) + } + Some(Value::Text(json_path)) => { + let selector = Selector::new(json_path.as_str()) + .map_err(|err| RadError::JsonPathParse { + description: err.to_string(), + })?; + let items: Vec = selector.find(&json_input) + .map(|item| into_radon_types(item)) + .collect(); + Ok(RadonArray::from(items)) + } + None => { + RadonTypes::try_from(json_input)?.try_into() + } + _ => Err(wrong_args()) + } +} +fn into_radon_types(value: &serde_json::Value) -> RadonTypes { + match value { + serde_json::Value::Number(value) => { + if value.is_f64() { + RadonTypes::from(RadonFloat::from(value.as_f64().unwrap_or_default())) + } else { + RadonTypes::from(RadonInteger::from(value.as_i64().unwrap_or_default() as i128)) + } + }, + serde_json::Value::Bool(value) => RadonTypes::from(RadonBoolean::from(*value)), + serde_json::Value::String(value) => RadonTypes::from(RadonString::from(value.clone())), + serde_json::Value::Object(entries) => { + let mut object: BTreeMap = BTreeMap::new(); + for (key, value) in entries { + object.insert(key.clone(), into_radon_types(value)); + } + RadonTypes::from(RadonMap::from(object)) + } + serde_json::Value::Array(values) => { + let items: Vec = values + .iter() + .map(|item| into_radon_types(item)) + .collect(); + RadonTypes::from(RadonArray::from(items)) + } + _ => RadonTypes::from(RadonError::new(RadError::JsonParse { description: value.to_string() })) + } } fn add_children( @@ -477,7 +557,7 @@ mod tests { #[test] fn test_parse_json_map() { let json_map = RadonString::from(r#"{ "Hello": "world" }"#); - let output = parse_json_map(&json_map).unwrap(); + let output = parse_json_map(&json_map, &None).unwrap(); let key = "Hello"; let value = RadonTypes::String(RadonString::from("world")); @@ -637,7 +717,7 @@ mod tests { fn test_parse_json_map_with_null_entries() { // When parsing a JSON map, any keys with value `null` are ignored let json_map = RadonString::from(r#"{ "Hello": "world", "Bye": null }"#); - let output = parse_json_map(&json_map).unwrap(); + let output = parse_json_map(&json_map, &None).unwrap(); let key = "Hello"; let value = RadonTypes::String(RadonString::from("world")); @@ -651,7 +731,7 @@ mod tests { #[test] fn test_parse_json_map_fail() { let invalid_json = RadonString::from(r#"{ "Hello": }"#); - let output = parse_json_map(&invalid_json).unwrap_err(); + let output = parse_json_map(&invalid_json, &None).unwrap_err(); let expected_err = RadError::JsonParse { description: "expected value at line 1 column 13".to_string(), @@ -659,7 +739,7 @@ mod tests { assert_eq!(output, expected_err); let json_array = RadonString::from(r#"[1,2,3]"#); - let output = parse_json_map(&json_array).unwrap_err(); + let output = parse_json_map(&json_array, &None).unwrap_err(); let expected_err = RadError::Decode { from: "cbor::value::Value", to: RadonMap::radon_type_name(), @@ -670,7 +750,7 @@ mod tests { #[test] fn test_parse_json_array() { let json_array = RadonString::from(r#"[1,2,3]"#); - let output = parse_json_array(&json_array).unwrap(); + let output = parse_json_array(&json_array, &None).unwrap(); let expected_output = RadonArray::from(vec![ RadonTypes::Integer(RadonInteger::from(1)), @@ -685,7 +765,7 @@ mod tests { fn test_parse_json_array_with_null_entries() { // When parsing a JSON array, any elements with value `null` are ignored let json_array = RadonString::from(r#"[null, 1, null, null, 2, 3, null]"#); - let output = parse_json_array(&json_array).unwrap(); + let output = parse_json_array(&json_array, &None).unwrap(); let expected_output = RadonArray::from(vec![ RadonTypes::Integer(RadonInteger::from(1)), @@ -699,7 +779,7 @@ mod tests { #[test] fn test_parse_json_array_fail() { let invalid_json = RadonString::from(r#"{ "Hello": }"#); - let output = parse_json_array(&invalid_json).unwrap_err(); + let output = parse_json_array(&invalid_json, &None).unwrap_err(); let expected_err = RadError::JsonParse { description: "expected value at line 1 column 13".to_string(), @@ -707,7 +787,7 @@ mod tests { assert_eq!(output, expected_err); let json_map = RadonString::from(r#"{ "Hello": "world" }"#); - let output = parse_json_array(&json_map).unwrap_err(); + let output = parse_json_array(&json_map, &None).unwrap_err(); let expected_err = RadError::Decode { from: "cbor::value::Value", to: RadonArray::radon_type_name(), @@ -1235,7 +1315,7 @@ mod tests { let args = vec![Value::Map(map)]; let result = string_match(&input_key, &args); - assert_eq!(result.unwrap_err().to_string(), "Wrong `RadonString::String match()` arguments: `[Map({Text(\"key1\"): Float(1.0), Text(\"key2\"): Float(2.0)})]`"); + assert_eq!(result.unwrap_err().to_string(), "Wrong `RadonString::StringMatch()` arguments: `[Map({Text(\"key1\"): Float(1.0), Text(\"key2\"): Float(2.0)})]`"); } #[test] diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index 5c3bed08f..ececf939d 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -131,7 +131,7 @@ impl Operable for RadonString { (RadonOpCodes::StringMatch, Some(args)) => { string_operators::string_match(self, args.as_slice()).map(RadonTypes::from) } - (RadonOpCodes::StringParseJSONArray, None) => string_operators::parse_json_array(self) + (RadonOpCodes::StringParseJSONArray, args) => string_operators::parse_json_array(self, &args) .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::StringParseJSONMap, args) => string_operators::parse_json_map(self, &args) From 25066131dd910053c2dcfd2e4c6c67e93c27493a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 10 Jan 2024 11:55:59 +0100 Subject: [PATCH 33/46] chore: cargo clippy --fix --- rad/src/operators/array.rs | 8 ++++---- rad/src/operators/bytes.rs | 4 ++-- rad/src/operators/string.rs | 10 +++++----- rad/src/types/string.rs | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index 94808287b..757a3432f 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -118,7 +118,7 @@ pub fn join( operator: "ArrayJoin".to_string(), }); } - let separator = if args.len() > 0 { + let separator = if !args.is_empty() { from_value::(args[0].to_owned()).unwrap_or_default() } else { String::from("") @@ -131,14 +131,14 @@ pub fn join( Ok(RadonTypes::from(RadonString::from(string_list.join(separator.as_str())))) } Some(first_item) => { - return Err(RadError::UnsupportedOperator { + Err(RadError::UnsupportedOperator { input_type: first_item.radon_type_name().to_string(), operator: "ArrayJoin".to_string(), args: Some(args.to_vec()) - }); + }) } _ => { - return Err(RadError::EmptyArray) + Err(RadError::EmptyArray) } } } diff --git a/rad/src/operators/bytes.rs b/rad/src/operators/bytes.rs index 0106eac5c..76a0dc3c6 100644 --- a/rad/src/operators/bytes.rs +++ b/rad/src/operators/bytes.rs @@ -66,12 +66,12 @@ pub fn stringify(input: &RadonBytes, args: &Option>) -> Result::default()).to_vec(), + args: args.to_owned().unwrap_or_default().to_vec(), }; let mut bytes_encoding = RadonBytesEncoding::Hex; match args { Some(args) => { - if args.len() > 0 { + if !args.is_empty() { let arg = args.first().ok_or_else(wrong_args)?.to_owned(); let bytes_encoding_u8 = from_value::(arg).map_err(|_| wrong_args())?; bytes_encoding = RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index 58d1faae8..e4adc233f 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -37,7 +37,7 @@ pub fn as_bytes(input: &RadonString, args: &Option>) -> Result::default()).to_vec(), + args: args.to_owned().unwrap_or_default().to_vec(), }; let mut input_string = input.value(); if input_string.starts_with("0x") { @@ -49,7 +49,7 @@ pub fn as_bytes(input: &RadonString, args: &Option>) -> Result { - if args.len() > 0 { + if !args.is_empty() { let arg = args.first().ok_or_else(wrong_args)?.to_owned(); let bytes_encoding_u8 = from_value::(arg).map_err(|_| wrong_args())?; bytes_encoding = RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; @@ -203,7 +203,7 @@ pub fn parse_json_array(input: &RadonString, args: &Option>) -> Resul description: err.to_string(), })?; let mut subitems: Vec = selector.find(&json_input) - .map(|item| into_radon_types(item)) + .map(into_radon_types) .collect(); if subitems.len() > 1 { items.insert(items.len(), RadonArray::from(subitems).into()); @@ -222,7 +222,7 @@ pub fn parse_json_array(input: &RadonString, args: &Option>) -> Resul description: err.to_string(), })?; let items: Vec = selector.find(&json_input) - .map(|item| into_radon_types(item)) + .map(into_radon_types) .collect(); Ok(RadonArray::from(items)) } @@ -254,7 +254,7 @@ fn into_radon_types(value: &serde_json::Value) -> RadonTypes { serde_json::Value::Array(values) => { let items: Vec = values .iter() - .map(|item| into_radon_types(item)) + .map(into_radon_types) .collect(); RadonTypes::from(RadonArray::from(items)) } diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index ececf939d..3148e5baf 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -131,10 +131,10 @@ impl Operable for RadonString { (RadonOpCodes::StringMatch, Some(args)) => { string_operators::string_match(self, args.as_slice()).map(RadonTypes::from) } - (RadonOpCodes::StringParseJSONArray, args) => string_operators::parse_json_array(self, &args) + (RadonOpCodes::StringParseJSONArray, args) => string_operators::parse_json_array(self, args) .map(RadonTypes::from) .map_err(Into::into), - (RadonOpCodes::StringParseJSONMap, args) => string_operators::parse_json_map(self, &args) + (RadonOpCodes::StringParseJSONMap, args) => string_operators::parse_json_map(self, args) .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) From d03ce84b6a97c8786d63e2bd8ad8d514d8d0839e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 12 Jan 2024 14:23:36 +0100 Subject: [PATCH 34/46] chore: cargo fmt --- rad/src/error.rs | 5 +- rad/src/lib.rs | 12 +-- rad/src/operators/array.rs | 31 ++++-- rad/src/operators/bytes.rs | 45 +++++---- rad/src/operators/integer.rs | 6 +- rad/src/operators/map.rs | 59 ++++++------ rad/src/operators/mod.rs | 2 +- rad/src/operators/string.rs | 179 +++++++++++++++++++---------------- rad/src/script.rs | 6 +- rad/src/types/bytes.rs | 6 +- rad/src/types/map.rs | 12 ++- rad/src/types/string.rs | 32 ++++--- 12 files changed, 226 insertions(+), 169 deletions(-) diff --git a/rad/src/error.rs b/rad/src/error.rs index c2c606ac7..3eeb11c3f 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -413,7 +413,10 @@ impl RadError { }) } } - None => Err(RadError::DecodeRadonErrorEmptyArray), + None => { + println!("None"); + Err(RadError::DecodeRadonErrorEmptyArray) + } } } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 4507a803e..c62706c34 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -299,11 +299,11 @@ async fn http_response( let mut response_bytes = Vec::::default(); // todo: before reading the response buffer, an error should be thrown if it was too big - body.read_to_end(&mut response_bytes).await.map_err(|x| { - RadError::HttpOther { + body.read_to_end(&mut response_bytes) + .await + .map_err(|x| RadError::HttpOther { message: x.to_string(), - } - })?; + })?; response = RadonTypes::from(RadonBytes::from(response_bytes)); } else { // response is a string @@ -323,8 +323,8 @@ async fn http_response( response = RadonTypes::from(RadonString::from(response_string)); } - let result = handle_response_with_data_report(retrieve, response, context, settings) - .map(|report| { + let result = + handle_response_with_data_report(retrieve, response, context, settings).map(|report| { let completion_ts = std::time::SystemTime::now(); RadonReport { context: ReportContext { diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index 757a3432f..468e0bf07 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -1,6 +1,6 @@ use std::{ clone::Clone, - convert::{TryFrom, TryInto} + convert::{TryFrom, TryInto}, }; use serde_cbor::value::{from_value, Value}; @@ -11,7 +11,10 @@ use crate::{ filters::{self, RadonFilters}, operators::string, reducers::{self, RadonReducers}, - script::{execute_radon_script, unpack_subscript, RadonScriptExecutionSettings, partial_results_extract}, + script::{ + execute_radon_script, partial_results_extract, unpack_subscript, + RadonScriptExecutionSettings, + }, types::{array::RadonArray, integer::RadonInteger, string::RadonString, RadonType, RadonTypes}, }; @@ -108,10 +111,7 @@ fn get_numeric_string(input: &RadonArray, args: &[Value]) -> Result Result { +pub fn join(input: &RadonArray, args: &[Value]) -> Result { // Join not applicable if the input array is not homogeneous if !input.is_homogeneous() { return Err(RadError::UnsupportedOpNonHomogeneous { @@ -140,6 +140,17 @@ pub fn join( _ => { Err(RadError::EmptyArray) } + .value() + .into_iter() + .map(|item| RadonString::try_from(item).unwrap_or_default().value()) + .collect(); + Ok(RadonTypes::from(RadonString::from( + string_list.join(separator.as_str()), + ))) + operator: "ArrayJoin".to_string(), + args: Some(args.to_vec()), + }), + _ => Err(RadError::EmptyArray), } } @@ -258,7 +269,7 @@ pub fn filter( pub fn pick( input: &RadonArray, args: &[Value], - _context: &mut ReportContext + _context: &mut ReportContext, ) -> Result { let not_found = |index: usize| RadError::ArrayIndexOutOfBounds { index: i32::try_from(index).unwrap(), @@ -286,7 +297,7 @@ pub fn pick( let index = from_value::(first_arg.clone()).map_err(|_| wrong_args())?; indexes.push(index); } - _ => return Err(wrong_args()) + _ => return Err(wrong_args()), }; } @@ -472,8 +483,8 @@ mod tests { operators::{ Operable, RadonOpCodes::{ - IntegerGreaterThan, IntegerMultiply, MapGetBoolean, MapGetFloat, MapGetInteger, - MapGetString, self, + self, IntegerGreaterThan, IntegerMultiply, MapGetBoolean, MapGetFloat, + MapGetInteger, MapGetString, }, }, types::{ diff --git a/rad/src/operators/bytes.rs b/rad/src/operators/bytes.rs index 76a0dc3c6..65ecce64f 100644 --- a/rad/src/operators/bytes.rs +++ b/rad/src/operators/bytes.rs @@ -5,7 +5,12 @@ use std::convert::TryFrom; use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, - types::{bytes::{RadonBytes, RadonBytesEncoding}, string::RadonString, integer::RadonInteger, RadonType}, + types::{ + bytes::{RadonBytes, RadonBytesEncoding}, + integer::RadonInteger, + string::RadonString, + RadonType, + }, }; pub fn as_integer(input: &RadonBytes) -> Result { @@ -13,12 +18,14 @@ pub fn as_integer(input: &RadonBytes) -> Result { match input_value_len { 1..=16 => { let mut bytes_array = [0u8; 16]; - bytes_array[16 - input_value_len ..].copy_from_slice(&input.value()); + bytes_array[16 - input_value_len..].copy_from_slice(&input.value()); Ok(RadonInteger::from(i128::from_be_bytes(bytes_array))) } - 17.. => Err(RadError::ParseInt { message: "Input buffer too big".to_string() }), - _ => Err(RadError::EmptyArray) - } + 17.. => Err(RadError::ParseInt { + message: "Input buffer too big".to_string(), + }), + _ => Err(RadError::EmptyArray), + } } pub fn hash(input: &RadonBytes, args: &[Value]) -> Result { @@ -43,17 +50,21 @@ pub fn length(input: &RadonBytes) -> RadonInteger { } pub fn slice(input: &RadonBytes, args: &[Value]) -> Result { - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "BytesSlice".to_string(), args: args.to_vec(), }; let end_index = input.value().len(); if end_index > 0 { - let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + let start_index = from_value::(args[0].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; let mut slice = input.value().as_slice().split_at(start_index).1.to_vec(); if args.len() == 2 { - let end_index = from_value::(args[1].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; + let end_index = from_value::(args[1].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; slice.truncate(end_index - start_index); } Ok(RadonBytes::from(slice)) @@ -63,7 +74,7 @@ pub fn slice(input: &RadonBytes, args: &[Value]) -> Result } pub fn stringify(input: &RadonBytes, args: &Option>) -> Result { - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "Stringify".to_string(), args: args.to_owned().unwrap_or_default().to_vec(), @@ -74,20 +85,18 @@ pub fn stringify(input: &RadonBytes, args: &Option>) -> Result(arg).map_err(|_| wrong_args())?; - bytes_encoding = RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; + bytes_encoding = + RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; } } - _ => () + _ => (), } match bytes_encoding { - RadonBytesEncoding::Hex => { - RadonString::try_from(Value::Text(hex::encode(input.value()))) - } - RadonBytesEncoding::Base64 => { - RadonString::try_from(Value::Text(base64::engine::general_purpose::STANDARD.encode(input.value()))) - } + RadonBytesEncoding::Hex => RadonString::try_from(Value::Text(hex::encode(input.value()))), + RadonBytesEncoding::Base64 => RadonString::try_from(Value::Text( + base64::engine::general_purpose::STANDARD.encode(input.value()), + )), } - } #[cfg(test)] diff --git a/rad/src/operators/integer.rs b/rad/src/operators/integer.rs index 49b3a4b00..5fda41039 100644 --- a/rad/src/operators/integer.rs +++ b/rad/src/operators/integer.rs @@ -5,8 +5,8 @@ use serde_cbor::value::{from_value, Value}; use crate::{ error::RadError, types::{ - boolean::RadonBoolean, float::RadonFloat, integer::RadonInteger, string::RadonString, - RadonType, bytes::RadonBytes, + boolean::RadonBoolean, bytes::RadonBytes, float::RadonFloat, integer::RadonInteger, + string::RadonString, RadonType, }, }; @@ -26,7 +26,7 @@ pub fn to_bytes(input: RadonInteger) -> Result { let mut leading_zeros = 0; for i in 0..bytes_array.len() { if bytes_array[i] != 0u8 { - break + break; } else { leading_zeros += 1; } diff --git a/rad/src/operators/map.rs b/rad/src/operators/map.rs index b4bd959bf..b399c2584 100644 --- a/rad/src/operators/map.rs +++ b/rad/src/operators/map.rs @@ -1,4 +1,4 @@ -use std::{convert::TryInto, collections::BTreeMap}; +use std::{collections::BTreeMap, convert::TryInto}; use serde_cbor::value::{from_value, Value}; use witnet_data_structures::radon_report::ReportContext; @@ -6,14 +6,17 @@ use witnet_data_structures::radon_report::ReportContext; use crate::{ error::RadError, operators::string, - types::{array::RadonArray, map::RadonMap, string::RadonString, RadonType, RadonTypes}, - script::{RadonScriptExecutionSettings, execute_radon_script, unpack_subscript, partial_results_extract}, + script::{ + execute_radon_script, partial_results_extract, unpack_subscript, + RadonScriptExecutionSettings, + }, + types::{array::RadonArray, map::RadonMap, string::RadonString, RadonType, RadonTypes}, }; pub fn alter( - input: &RadonMap, + input: &RadonMap, args: &[Value], - context: &mut ReportContext + context: &mut ReportContext, ) -> Result { let wrong_args = || RadError::WrongArguments { input_type: RadonMap::radon_type_name(), @@ -33,7 +36,7 @@ pub fn alter( Value::Text(key) => { input_keys.push(key.clone()); } - _ => return Err(wrong_args()) + _ => return Err(wrong_args()), }; let subscript = args.get(1).ok_or_else(wrong_args)?; @@ -45,24 +48,22 @@ pub fn alter( inner: Box::new(e), }; let subscript = unpack_subscript(subscript).map_err(subscript_err)?; - - let not_found = |key_str: &str| RadError::MapKeyNotFound { key: String::from(key_str) }; + + let not_found = |key_str: &str| RadError::MapKeyNotFound { + key: String::from(key_str), + }; let input_map = input.value(); let mut output_map = input.value().clone(); let mut reports = vec![]; - + let settings = RadonScriptExecutionSettings::tailored_to_stage(&context.stage); for key in input_keys { let value = input_map .get(key.as_str()) .ok_or_else(|| not_found(key.as_str()))?; - let report = execute_radon_script( - value.clone(), - subscript.as_slice(), - context, - settings - )?; + let report = + execute_radon_script(value.clone(), subscript.as_slice(), context, settings)?; // If there is an error while altering value, short-circuit and bubble up the error as it comes // from the radon script execution if let RadonTypes::RadonError(error) = &report.result { @@ -71,14 +72,14 @@ pub fn alter( output_map.insert(key, report.result.clone()); } reports.push(report); - } - + } + // Extract the partial results from the reports and put them in the execution context if needed partial_results_extract(&subscript, &reports, context); Ok(RadonMap::from(output_map)) } - _ => Err(wrong_args()) + _ => Err(wrong_args()), } } @@ -156,10 +157,10 @@ pub fn values(input: &RadonMap) -> RadonArray { } pub fn pick(input: &RadonMap, args: &[Value]) -> Result { - let not_found = |key_str: &str| RadError::MapKeyNotFound { - key: String::from(key_str) + let not_found = |key_str: &str| RadError::MapKeyNotFound { + key: String::from(key_str), }; - + let wrong_args = || RadError::WrongArguments { input_type: RadonMap::radon_type_name(), operator: "Pick".to_string(), @@ -174,14 +175,15 @@ pub fn pick(input: &RadonMap, args: &[Value]) -> Result { match first_arg { Value::Array(keys) => { for key in keys.iter() { - let key_string = from_value::(key.to_owned()).map_err(|_| wrong_args())?; + let key_string = + from_value::(key.to_owned()).map_err(|_| wrong_args())?; input_keys.push(key_string); } } Value::Text(key) => { input_keys.push(key.clone()); } - _ => return Err(wrong_args()) + _ => return Err(wrong_args()), }; } @@ -190,18 +192,17 @@ pub fn pick(input: &RadonMap, args: &[Value]) -> Result { if let Some(value) = input.value().get(&key) { output_map.insert(key, value.clone()); } else { - return Err(not_found(key.as_str())) + return Err(not_found(key.as_str())); } } Ok(RadonMap::from(output_map)) } pub fn stringify(input: &RadonMap) -> Result { - let json_string = serde_json::to_string(&input.value()) - .map_err(|_| RadError::Decode { - from: "RadonMap", - to: "RadonString" - })?; + let json_string = serde_json::to_string(&input.value()).map_err(|_| RadError::Decode { + from: "RadonMap", + to: "RadonString", + })?; Ok(RadonString::from(json_string)) } diff --git a/rad/src/operators/mod.rs b/rad/src/operators/mod.rs index 9afdbb399..f169c2e2d 100644 --- a/rad/src/operators/mod.rs +++ b/rad/src/operators/mod.rs @@ -48,7 +48,7 @@ pub enum RadonOpCodes { BooleanNegate = 0x22, BooleanToString = 0x20, /////////////////////////////////////////////////////////////////////// - // Bytes operator codes (start at 0x30) + // Bytes operator codes (start at 0x30) BytesAsInteger = 0x32, BytesHash = 0x31, BytesLength = 0x34, diff --git a/rad/src/operators/string.rs b/rad/src/operators/string.rs index e4adc233f..b05b28f49 100644 --- a/rad/src/operators/string.rs +++ b/rad/src/operators/string.rs @@ -5,20 +5,26 @@ use std::{ }; use base64::Engine; +use jsonpath::Selector; use serde_cbor::value::{from_value, Value}; use serde_json::Value as JsonValue; -use jsonpath::Selector; -use slicestring::Slice; use regex::Regex; +use slicestring::Slice; use witnet_data_structures::radon_error::RadonError; use crate::{ error::RadError, hash_functions::{self, RadonHashFunctions}, types::{ - array::RadonArray, boolean::RadonBoolean, bytes::{RadonBytes, RadonBytesEncoding}, float::RadonFloat, - integer::RadonInteger, map::RadonMap, string::RadonString, RadonType, RadonTypes, + array::RadonArray, + boolean::RadonBoolean, + bytes::{RadonBytes, RadonBytesEncoding}, + float::RadonFloat, + integer::RadonInteger, + map::RadonMap, + string::RadonString, + RadonType, RadonTypes, }, }; @@ -34,7 +40,7 @@ pub fn as_bool(input: &RadonString) -> Result { } pub fn as_bytes(input: &RadonString, args: &Option>) -> Result { - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "AsBytes".to_string(), args: args.to_owned().unwrap_or_default().to_vec(), @@ -52,30 +58,27 @@ pub fn as_bytes(input: &RadonString, args: &Option>) -> Result(arg).map_err(|_| wrong_args())?; - bytes_encoding = RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; + bytes_encoding = + RadonBytesEncoding::try_from(bytes_encoding_u8).map_err(|_| wrong_args())?; } } - _ => () + _ => (), } match bytes_encoding { - RadonBytesEncoding::Hex => { - Ok(RadonBytes::from( - hex::decode(input_string.as_str()) + RadonBytesEncoding::Hex => Ok(RadonBytes::from( + hex::decode(input_string.as_str()).map_err(|_err| RadError::Decode { + from: "RadonString", + to: "RadonBytes", + })?, + )), + RadonBytesEncoding::Base64 => Ok(RadonBytes::from( + base64::engine::general_purpose::STANDARD + .decode(input.value()) .map_err(|_err| RadError::Decode { from: "RadonString", to: "RadonBytes", - })? - )) - } - RadonBytesEncoding::Base64 => { - Ok(RadonBytes::from( - base64::engine::general_purpose::STANDARD.decode(input.value()) - .map_err(|_err| RadError::Decode { - from: "RadonString", - to: "RadonBytes" - })? - )) - } + })?, + )), } } @@ -147,64 +150,69 @@ pub fn parse_json(input: &RadonString) -> Result { RadonTypes::try_from(json_value) } -pub fn parse_json_map(input: &RadonString, args: &Option>) -> Result { - let not_found = |json_path: &str| RadError::JsonPathNotFound { - path: String::from(json_path) +pub fn parse_json_map( + input: &RadonString, + args: &Option>, +) -> Result { + let not_found = |json_path: &str| RadError::JsonPathNotFound { + path: String::from(json_path), }; - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "ParseJsonMap".to_string(), args: args.to_owned().unwrap_or_default(), }; - let json_input: JsonValue = serde_json::from_str(&input.value()) - .map_err(|err| RadError::JsonParse { - description: err.to_string(), - })?; - + let json_input: JsonValue = + serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse { + description: err.to_string(), + })?; + match args.to_owned().unwrap_or_default().get(0) { Some(Value::Text(json_path)) => { - let selector = Selector::new(json_path.as_str()) - .map_err(|err| RadError::JsonPathParse { + let selector = + Selector::new(json_path.as_str()).map_err(|err| RadError::JsonPathParse { description: err.to_string(), })?; - let item = selector.find(&json_input) + let item = selector + .find(&json_input) .next() .ok_or_else(|| not_found(json_path.as_str()))?; RadonTypes::try_from(item.to_owned())?.try_into() - }, - None => { - RadonTypes::try_from(json_input)?.try_into() - }, - _ => Err(wrong_args()) + } + None => RadonTypes::try_from(json_input)?.try_into(), + _ => Err(wrong_args()), } } -pub fn parse_json_array(input: &RadonString, args: &Option>) -> Result { - let wrong_args = || RadError::WrongArguments { +pub fn parse_json_array( + input: &RadonString, + args: &Option>, +) -> Result { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "ParseJsonArray".to_string(), args: args.to_owned().unwrap_or_default(), }; - let json_input: JsonValue = serde_json::from_str(&input.value()) - .map_err(|err| RadError::JsonParse { - description: err.to_string(), - })?; + let json_input: JsonValue = + serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse { + description: err.to_string(), + })?; match args.to_owned().unwrap_or_default().get(0) { Some(Value::Array(values)) => { let mut items: Vec = vec![]; for path in values { if let Value::Text(json_path) = path { - let selector = Selector::new(json_path.as_str()) - .map_err(|err| RadError::JsonPathParse { + let selector = Selector::new(json_path.as_str()).map_err(|err| { + RadError::JsonPathParse { description: err.to_string(), - })?; - let mut subitems: Vec = selector.find(&json_input) - .map(into_radon_types) - .collect(); + } + })?; + let mut subitems: Vec = + selector.find(&json_input).map(into_radon_types).collect(); if subitems.len() > 1 { items.insert(items.len(), RadonArray::from(subitems).into()); } else { @@ -217,19 +225,15 @@ pub fn parse_json_array(input: &RadonString, args: &Option>) -> Resul Ok(RadonArray::from(items)) } Some(Value::Text(json_path)) => { - let selector = Selector::new(json_path.as_str()) - .map_err(|err| RadError::JsonPathParse { + let selector = + Selector::new(json_path.as_str()).map_err(|err| RadError::JsonPathParse { description: err.to_string(), })?; - let items: Vec = selector.find(&json_input) - .map(into_radon_types) - .collect(); + let items: Vec = selector.find(&json_input).map(into_radon_types).collect(); Ok(RadonArray::from(items)) } - None => { - RadonTypes::try_from(json_input)?.try_into() - } - _ => Err(wrong_args()) + None => RadonTypes::try_from(json_input)?.try_into(), + _ => Err(wrong_args()), } } @@ -239,9 +243,11 @@ fn into_radon_types(value: &serde_json::Value) -> RadonTypes { if value.is_f64() { RadonTypes::from(RadonFloat::from(value.as_f64().unwrap_or_default())) } else { - RadonTypes::from(RadonInteger::from(value.as_i64().unwrap_or_default() as i128)) + RadonTypes::from(RadonInteger::from( + value.as_i64().unwrap_or_default() as i128 + )) } - }, + } serde_json::Value::Bool(value) => RadonTypes::from(RadonBoolean::from(*value)), serde_json::Value::String(value) => RadonTypes::from(RadonString::from(value.clone())), serde_json::Value::Object(entries) => { @@ -252,13 +258,12 @@ fn into_radon_types(value: &serde_json::Value) -> RadonTypes { RadonTypes::from(RadonMap::from(object)) } serde_json::Value::Array(values) => { - let items: Vec = values - .iter() - .map(into_radon_types) - .collect(); + let items: Vec = values.iter().map(into_radon_types).collect(); RadonTypes::from(RadonArray::from(items)) } - _ => RadonTypes::from(RadonError::new(RadError::JsonParse { description: value.to_string() })) + _ => RadonTypes::from(RadonError::new(RadError::JsonParse { + description: value.to_string(), + })), } } @@ -372,20 +377,22 @@ pub fn radon_trim(input: &RadonString) -> String { } } - pub fn replace(input: &RadonString, args: &[Value]) -> Result { - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "StringReplace".to_string(), args: args.to_vec(), }; let regex = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; let replacement = RadonString::try_from(args.get(1).ok_or_else(wrong_args)?.to_owned())?; - Ok(RadonString::from(input.value().as_str().replace(regex.value().as_str(), replacement.value().as_str()))) + Ok(RadonString::from(input.value().as_str().replace( + regex.value().as_str(), + replacement.value().as_str(), + ))) } pub fn slice(input: &RadonString, args: &[Value]) -> Result { - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "StringSlice".to_string(), args: args.to_vec(), @@ -393,26 +400,40 @@ pub fn slice(input: &RadonString, args: &[Value]) -> Result { - let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; - end_index = from_value::(args[1].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; - Ok(RadonString::from(input.value().as_str().slice(start_index..end_index))) + let start_index = from_value::(args[0].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + end_index = from_value::(args[1].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + Ok(RadonString::from( + input.value().as_str().slice(start_index..end_index), + )) } 1 => { - let start_index = from_value::(args[0].clone()).unwrap_or_default().rem_euclid(end_index as i64) as usize; - Ok(RadonString::from(input.value().as_str().slice(start_index..end_index))) + let start_index = from_value::(args[0].clone()) + .unwrap_or_default() + .rem_euclid(end_index as i64) as usize; + Ok(RadonString::from( + input.value().as_str().slice(start_index..end_index), + )) } - _ => Err(wrong_args()) + _ => Err(wrong_args()), } } pub fn split(input: &RadonString, args: &[Value]) -> Result { - let wrong_args = || RadError::WrongArguments { + let wrong_args = || RadError::WrongArguments { input_type: RadonString::radon_type_name(), operator: "StringSplit".to_string(), args: args.to_vec(), }; let pattern = RadonString::try_from(args.first().ok_or_else(wrong_args)?.to_owned())?; - let parts: Vec = Regex::new(pattern.value().as_str()).unwrap().split(input.value().as_str()).map(|part| RadonTypes::from(RadonString::from(part))).collect(); + let parts: Vec = Regex::new(pattern.value().as_str()) + .unwrap() + .split(input.value().as_str()) + .map(|part| RadonTypes::from(RadonString::from(part))) + .collect(); Ok(RadonArray::from(parts)) } diff --git a/rad/src/script.rs b/rad/src/script.rs index a203ef477..33150cc66 100644 --- a/rad/src/script.rs +++ b/rad/src/script.rs @@ -1,8 +1,4 @@ -use std::{ - clone::Clone, - convert::TryFrom, - iter, -}; +use std::{clone::Clone, convert::TryFrom, iter}; use serde_cbor::{ self as cbor, diff --git a/rad/src/types/bytes.rs b/rad/src/types/bytes.rs index 6f9637f01..03f02f6e2 100644 --- a/rad/src/types/bytes.rs +++ b/rad/src/types/bytes.rs @@ -104,9 +104,9 @@ impl Operable for RadonBytes { (RadonOpCodes::BytesLength, None) => { Ok(RadonTypes::from(bytes_operators::length(self))) } - (RadonOpCodes::BytesSlice, Some(args)) => { - bytes_operators::slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) - }, + (RadonOpCodes::BytesSlice, Some(args)) => bytes_operators::slice(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into), (RadonOpCodes::BytesStringify, args) => bytes_operators::stringify(self, args) .map(RadonTypes::from) .map_err(Into::into), diff --git a/rad/src/types/map.rs b/rad/src/types/map.rs index c86f294a1..ec9963d15 100644 --- a/rad/src/types/map.rs +++ b/rad/src/types/map.rs @@ -161,9 +161,15 @@ impl Operable for RadonMap { } (RadonOpCodes::MapKeys, None) => Ok(RadonTypes::from(map_operators::keys(self))), (RadonOpCodes::MapValues, None) => Ok(RadonTypes::from(map_operators::values(self))), - (RadonOpCodes::MapAlter, Some(args)) => map_operators::alter(self, args, context).map(RadonTypes::from), - (RadonOpCodes::MapPick, Some(args)) => map_operators::pick(self, args).map(RadonTypes::from), - (RadonOpCodes::MapStringify, None) => map_operators::stringify(self).map(RadonTypes::from), + (RadonOpCodes::MapAlter, Some(args)) => { + map_operators::alter(self, args, context).map(RadonTypes::from) + } + (RadonOpCodes::MapPick, Some(args)) => { + map_operators::pick(self, args).map(RadonTypes::from) + } + (RadonOpCodes::MapStringify, None) => { + map_operators::stringify(self).map(RadonTypes::from) + } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_MAP_TYPE_NAME.to_string(), operator: op_code.to_string(), diff --git a/rad/src/types/string.rs b/rad/src/types/string.rs index 3148e5baf..be0f66982 100644 --- a/rad/src/types/string.rs +++ b/rad/src/types/string.rs @@ -126,17 +126,21 @@ impl Operable for RadonString { .map(RadonTypes::from) .map_err(Into::into), (RadonOpCodes::StringLength, None) => { - Ok(RadonTypes::from(string_operators::length(self))) - } + Ok(RadonTypes::from(string_operators::length(self))) + } (RadonOpCodes::StringMatch, Some(args)) => { string_operators::string_match(self, args.as_slice()).map(RadonTypes::from) } - (RadonOpCodes::StringParseJSONArray, args) => string_operators::parse_json_array(self, args) - .map(RadonTypes::from) - .map_err(Into::into), - (RadonOpCodes::StringParseJSONMap, args) => string_operators::parse_json_map(self, args) - .map(RadonTypes::from) - .map_err(Into::into), + (RadonOpCodes::StringParseJSONArray, args) => { + string_operators::parse_json_array(self, args) + .map(RadonTypes::from) + .map_err(Into::into) + } + (RadonOpCodes::StringParseJSONMap, args) => { + string_operators::parse_json_map(self, args) + .map(RadonTypes::from) + .map_err(Into::into) + } (RadonOpCodes::StringParseXMLMap, None) => string_operators::parse_xml_map(self) .map(RadonTypes::from) .map_err(Into::into), @@ -147,13 +151,19 @@ impl Operable for RadonString { Ok(RadonTypes::from(string_operators::to_uppercase(self))) } (RadonOpCodes::StringReplace, Some(args)) => { - string_operators::replace(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + string_operators::replace(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into) } (RadonOpCodes::StringSlice, Some(args)) => { - string_operators::slice(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + string_operators::slice(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into) } (RadonOpCodes::StringSplit, Some(args)) => { - string_operators::split(self, args.as_slice()).map(RadonTypes::from).map_err(Into::into) + string_operators::split(self, args.as_slice()) + .map(RadonTypes::from) + .map_err(Into::into) } (op_code, args) => Err(RadError::UnsupportedOperator { input_type: RADON_STRING_TYPE_NAME.to_string(), From d066a4dcf5a2c4599e36d869169642f956ef6a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Fri, 12 Jan 2024 14:26:02 +0100 Subject: [PATCH 35/46] fix: failing radon error tests --- rad/src/error.rs | 4 ++++ rad/src/operators/array.rs | 21 +++++---------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/rad/src/error.rs b/rad/src/error.rs index 3eeb11c3f..e27834450 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -564,6 +564,7 @@ impl RadError { }; Some(serialize_args((message,))?) } + RadError::BufferIsNotValue { description } => Some(serialize_args((description,))?), _ => None, }; @@ -770,6 +771,9 @@ mod tests { inner: None, message: Some("Only the message field is serialized".to_string()), }, + RadonErrors::BufferIsNotValue => RadError::BufferIsNotValue { + description: "Buffer is no value".to_string(), + }, // If this panics after adding a new `RadonTypes`, add a new example above _ => panic!("No example for {:?}", radon_errors), } diff --git a/rad/src/operators/array.rs b/rad/src/operators/array.rs index 468e0bf07..1f3c809cd 100644 --- a/rad/src/operators/array.rs +++ b/rad/src/operators/array.rs @@ -123,23 +123,9 @@ pub fn join(input: &RadonArray, args: &[Value]) -> Result } else { String::from("") }; - match input.value().first() { + match input.value().first() { Some(RadonTypes::String(_)) => { - let string_list: Vec = input.value().into_iter().map(|item| - RadonString::try_from(item).unwrap_or_default().value() - ).collect(); - Ok(RadonTypes::from(RadonString::from(string_list.join(separator.as_str())))) - } - Some(first_item) => { - Err(RadError::UnsupportedOperator { - input_type: first_item.radon_type_name().to_string(), - operator: "ArrayJoin".to_string(), - args: Some(args.to_vec()) - }) - } - _ => { - Err(RadError::EmptyArray) - } + let string_list: Vec = input .value() .into_iter() .map(|item| RadonString::try_from(item).unwrap_or_default().value()) @@ -147,6 +133,9 @@ pub fn join(input: &RadonArray, args: &[Value]) -> Result Ok(RadonTypes::from(RadonString::from( string_list.join(separator.as_str()), ))) + } + Some(first_item) => Err(RadError::UnsupportedOperator { + input_type: first_item.radon_type_name().to_string(), operator: "ArrayJoin".to_string(), args: Some(args.to_vec()), }), From cb9c3a0f869de74e7d923197defdd651ec851237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 10:49:07 +0100 Subject: [PATCH 36/46] chore: add new RadonErrors cases and order them all alphabetically --- data_structures/src/radon_error.rs | 144 ++++++++++++++++++++++------- 1 file changed, 110 insertions(+), 34 deletions(-) diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs index 6fdf30d26..78171cb4d 100644 --- a/data_structures/src/radon_error.rs +++ b/data_structures/src/radon_error.rs @@ -6,6 +6,8 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use serde::Serialize; use serde_cbor::Value as SerdeCborValue; +use crate::chain::tapi::ActiveWips; + #[derive(Clone, Copy, Debug, Eq, IntoPrimitive, PartialEq, Serialize, TryFromPrimitive)] #[repr(u8)] /// List of RADON-level errors. @@ -14,64 +16,138 @@ use serde_cbor::Value as SerdeCborValue; pub enum RadonErrors { /// Unknown error. Something went really bad! Unknown = 0x00, - // Script format errors + + /////////////////////////////////////////////////////////////////////////// + // Script format error sub-codes /// At least one of the source scripts is not a valid CBOR-encoded value. SourceScriptNotCBOR = 0x01, /// The CBOR value decoded from a source script is not an Array. SourceScriptNotArray = 0x02, /// The Array value decoded form a source script is not a valid RADON script. SourceScriptNotRADON = 0x03, - // Complexity errors + /// The request body of at least one data source was not properly formated. + SourceRequestBody = 0x04, + /// The request headers of at least one data source was not properly formated. + SourceRequestHeaders = 0x05, + /// The request URL of at least one data source was not properly formated. + SourceRequestURL = 0x06, + + /////////////////////////////////////////////////////////////////////////// + // Complexity error sub-codes /// The request contains too many sources. RequestTooManySources = 0x10, /// The script contains too many calls. ScriptTooManyCalls = 0x11, - // Operator errors - /// The operator does not exist. + + /////////////////////////////////////////////////////////////////////////// + // Lack of support error sub-codes + /// Some Radon operator opcode is not currently supported. UnsupportedOperator = 0x20, - // Retrieval-specific errors - /// At least one of the sources could not be retrieved, but returned HTTP error. - HTTPError = 0x30, - /// Al least one of the sources could not be retrieved, timeout reached. - RetrieveTimeout = 0x31, - /// Value cannot be extracted from binary buffer - BufferIsNotValue = 0x32, - // Math errors + /// Some Radon filter opcode is not currently supported. + UnsupportedFilter = 0x21, + /// Some Radon hash function is not currently supported. + UnsupportedHashFunction = 0x22, + /// Some Radon reducer opcode is not currently supported. + UnsupportedReducer = 0x23, + /// Some Radon request type is not currently supported. + UnsupportedRequestType = 0x24, + /// Some Radon encoding function is not currently supported. + UnsupportedEncodingFunction = 0x25, + /// Wrong number (or type) of arguments were passed to some Radon operator. + WrongArguments = 0x28, + + /////////////////////////////////////////////////////////////////////////// + // Retrieval-specific error sub-codes + /// A majority of data sources returned an HTTP status code other than 200. + HttpErrors = 0x30, + /// A majority of data sources timed out. + RetrievalsTimeout = 0x31, + + /////////////////////////////////////////////////////////////////////////// + // Script-specific error sub-codes /// Math operator caused an underflow. - Underflow = 0x40, + MathUnderflow = 0x40, /// Math operator caused an overflow. - Overflow = 0x41, + MathOverflow = 0x41, /// Tried to divide by zero. - DivisionByZero = 0x42, - // Other errors - /// Received zero reveals - NoReveals = 0x50, - /// Insufficient consensus in tally precondition clause - InsufficientConsensus = 0x51, - /// Received zero commits + MathDivisionByZero = 0x42, + /// Wrong input to subscript call. + WrongSubscriptInput = 0x43, + /// Value cannot be extracted from input binary buffer. + BufferIsNotValue = 0x44, + /// Value cannot be decoded from expected type. + Decode = 0x45, + /// Unexpected empty array. + EmptyArray = 0x46, + /// Value cannot be encoded to expected type. + Encode = 0x47, + /// Failed to filter input values. + Filter = 0x48, + /// Failed to hash input value. + Hash = 0x49, + /// Mismatching array ranks. + MismatchingArrays = 0x4A, + /// Failed to process non-homogenous array. + NonHomogeneousArrays = 0x4B, + /// Failed to parse syntax of some input value, or argument. + Parse = 0x4C, + /// Parsing logic limits were exceeded. + ParseOverflow = 0x4D, + /// Failed to reduce input values. + Reduce = 0x4E, + + /////////////////////////////////////////////////////////////////////////// + // Actual result first-order error codes that can be included in a Tally. + /// Not enough reveal quorum was reached on tally stage. + InsufficientQuorum = 0x50, + /// No actual reveal majority was reached on tally stage. + InsufficientMajority = 0x51, + /// Not enough commits were received before tally stage. InsufficientCommits = 0x52, - /// Generic error during tally execution + /// Generic error during tally execution. TallyExecution = 0x53, - /// Invalid reveal serialization (malformed reveals are converted to this value) - MalformedReveal = 0x60, - /// Failed to encode reveal - EncodeReveal = 0x61, - // Access errors - /// Tried to access a value from an index using an index that is out of bounds + /// Some data sources could either be temporarily unresponsive or failing to report the requested data: + CircumstantialFailure = 0x54, + /// At least one data source is inconsistent when queried through multiple transports at once: + InconsistentSources = 0x55, + /// Values returned from a majority of data sources did not match the expected schema: + MalformedResponses = 0x56, + /// The data request was not properly formated: + MalformedDataRequest = 0x57, + /// The size of serialized tally result exceeds allowance: + OversizedTallyResult = 0x5F, + + + /////////////////////////////////////////////////////////////////////////// + // Inter-stage runtime error sub-codes + /// Data aggregation reveals could not get decoded on tally stage: + MalformedReveals = 0x60, + /// The result to data aggregation could not get encoded: + EncodeReveals = 0x61, + /// A mode tie ocurred when calculating the mode value on aggregation stage: + ModeTie = 0x62, + + /////////////////////////////////////////////////////////////////////////// + // Runtime access error sub-codes + /// Tried to access a value from an index using an index that is out of bounds. ArrayIndexOutOfBounds = 0x70, - /// Tried to access a value from a map using a key that does not exist + /// Tried to access a value from a map using a key that does not exist. MapKeyNotFound = 0x71, - // Bridge errors: errors that only belong in inter-client communication - /// Requests that cannot be parsed must always get this error as their result. - /// However, this is not a valid result in a Tally transaction, because invalid requests - /// are never included into blocks and therefore never get a Tally in response. + /// Tried to extract value from a map using a JSON Path that returns no values. + JsonPathNotFound = 0x72, + + /////////////////////////////////////////////////////////////////////////// + // Inter-client first-order error codes. + /// Requests that cannot be relayed into the Witnet blockchain should be reported + /// with one of these errors. BridgeMalformedRequest = 0xE0, /// The request is rejected on the grounds that it may cause the submitter to spend or stake an /// amount of value that is unjustifiably high when compared with the reward they will be getting BridgePoorIncentives = 0xE1, /// The request result length exceeds a bridge contract defined limit BridgeOversizedResult = 0xE2, - // This should not exist: + + // This should never happen: /// Some tally error is not intercepted but should UnhandledIntercept = 0xFF, } From 5c88ee07cff3f30a50f80a9cc5f5fc8692f6b2a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 11:00:36 +0100 Subject: [PATCH 37/46] chore: add new RadonErrors cases and order them all alphabetically --- rad/src/error.rs | 546 ++++++++++++++++++++++++++--------------------- 1 file changed, 304 insertions(+), 242 deletions(-) diff --git a/rad/src/error.rs b/rad/src/error.rs index e27834450..25d641653 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -14,66 +14,198 @@ use crate::{operators::RadonOpCodes, types::array::RadonArray}; /// RAD errors. #[derive(Clone, Debug, Fail, PartialEq)] pub enum RadError { - /// An unknown error. Something went really bad! - #[fail(display = "Unknown error")] - Unknown, + + /// The given subscript does not return RadonBoolean in an ArrayFilter + #[fail( + display = "ArrayFilter subscript output was not RadonBoolean (was `{}`)", + value + )] + ArrayFilterWrongSubscript { value: String }, + /// The given index is not present in a RadonArray + #[fail(display = "Failed to get item at index `{}` from RadonArray", index)] + ArrayIndexOutOfBounds { index: i32 }, + /// Subscripts should be an array + #[fail(display = "Subscript should be an array but is: {:?}", value)] + BadSubscriptFormat { value: SerdeCborValue }, + /// Failed to parse a Value from a buffer + #[fail( + display = "Failed to parse a Value from a buffer. Error message: {}", + description + )] + BufferIsNotValue { description: String }, /// Failed to decode a type from other #[fail(display = "Failed to decode {} from {}", to, from)] Decode { from: &'static str, to: &'static str, }, + /// Alleged `RadonError` has a `RadonTypes` argument which was wrongly serialized + // FIXME(#953): this error should not exist, but it is useful to detect problems with the + // current hacky implementation + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its arguments (`{:?}`) were not compatible: {}", + arguments, message + )] + DecodeRadonErrorArgumentsRadonTypesFail { + arguments: Option>, + message: String, + }, + /// Alleged `RadonError` contains an error code that is not `u8` + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its first element (the error code) was not `cbor::value::Value::U8` (was actually `{}`)", + actual_type + )] + DecodeRadonErrorBadCode { actual_type: String }, + /// Alleged `RadonError` is actually an empty `cbor::value::Value::Array` + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because the array was empty" + )] + DecodeRadonErrorEmptyArray, + /// Alleged `RadonError` does not have any arguments + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its arguments are empty" + )] + DecodeRadonErrorMissingArguments, + /// Alleged `RadonError` is actually not an instance of `cbor::value::Value::Array` + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value` that was not `Array` (was actually `{}`)", + actual_type + )] + DecodeRadonErrorNotArray { actual_type: String }, + /// Alleged `RadonError` contains an unknown error code + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its first element (`{:?}`) did not match any known error code", + error_code + )] + DecodeRadonErrorUnknownCode { error_code: u8 }, + /// Alleged `RadonError` does not have the expected arguments + #[fail( + display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its arguments (`{:?}`) were not compatible: {}", + arguments, message + )] + DecodeRadonErrorWrongArguments { + arguments: Option, + message: String, + }, + /// Arrays to be reduced have different sizes + #[fail( + display = "Arrays to be reduced in {} have different sizes. {} != {}", + method, first, second + )] + DifferentSizeArrays { + method: String, + first: usize, + second: usize, + }, + /// Tried to apply mode reducer on an empty array + #[fail(display = "Tried to apply mode reducer on an empty array")] + EmptyArray, /// Failed to encode a type into other #[fail(display = "Failed to encode {} into {}", from, to)] Encode { from: &'static str, to: &'static str, }, + /// Failed to encode `RadonError` arguments + #[fail(display = "Failed to encode `RadonError` arguments `{}`", error_args)] + EncodeRadonErrorArguments { error_args: String }, + /// `RadError` cannot be converted to `RadonError` because the error code is not defined + #[fail( + display = "`RadError` cannot be converted to `RadonError` because the error code is not defined" + )] + EncodeRadonErrorUnknownCode, + /// Failed to encode reveal + #[fail(display = "Error when encoding reveal value")] + EncodeReveal, /// Failed to calculate the hash of a RADON value or structure #[fail(display = "Failed to calculate the hash of a RADON value or structure")] Hash, + /// Failed to execute HTTP request + #[fail( + display = "Failed to execute HTTP request with error message: {}", + message + )] + HttpOther { message: String }, + /// The HTTP response was an error code + #[fail(display = "HTTP GET response was an HTTP error code: {}", status_code)] + HttpStatus { status_code: u16 }, + /// Source looks inconsistent when queried through multiple transports at once. + #[fail(display = "Source looks inconsistent when queried through multiple transports at once")] + InconsistentSource, + /// Error while parsing HTTP header + #[fail( + display = "invalid HTTP body: {}", + error + )] + InvalidHttpBody { error: String }, + /// Error while parsing HTTP header + #[fail( + display = "invalid HTTP header: {}. name={:?}, value={:?}", + error, name, value + )] + InvalidHttpHeader { + name: String, + value: String, + error: String, + }, + /// Error while parsing HTTP response + #[fail( + display = "invalid HTTP response: {}", + error + )] + InvalidHttpResponse { error: String }, + /// Invalid script + #[fail( + display = "CBOR value cannot be translated into a proper RADON script: {:?}", + value + )] + InvalidScript { value: SerdeCborValue }, /// Failed to parse an object from a JSON buffer #[fail( display = "Failed to parse an object from a JSON buffer: {:?}", description )] JsonParse { description: String }, - /// The given JSON path is not present in a JSON-stringified object - #[fail(display = "Failed to find JSON path `{}` from RadonString", path)] - JsonPathNotFound { path: String }, /// Failed to parse a JSON path selector from a string value #[fail( display = "Failed to parse a JSON path from a string value: {:?}", description )] JsonPathParse { description: String }, - /// Failed to parse an object from a XML buffer - #[fail( - display = "Failed to parse an object from a XML buffer: {:?}", - description - )] - XmlParse { description: String }, - /// Failed to parse an object from a XML buffer by depth overflow - #[fail(display = "Failed to parse an object from a XML buffer: XML depth overflow")] - XmlParseOverflow, - /// The given index is not present in a RadonArray - #[fail(display = "Failed to get item at index `{}` from RadonArray", index)] - ArrayIndexOutOfBounds { index: i32 }, + /// The given JSON path is not present in a JSON-stringified object + #[fail(display = "Failed to find JSON path `{}` from RadonString", path)] + JsonPathNotFound { path: String }, + /// Invalid reveal serialization (malformed reveals are converted to this value) + #[fail(display = "The reveal was not serialized correctly")] + MalformedReveal, /// The given key is not present in a RadonMap #[fail(display = "Failed to get key `{}` from RadonMap", key)] MapKeyNotFound { key: String }, - /// The given subscript does not return RadonBoolean in an ArrayFilter + /// Tried to divide by zero. + #[fail(display = "Tried to divide by zero")] + MathDivisionByZero, + /// Overflow error + #[fail(display = "Overflow error")] + MathOverflow, + /// Math operator caused an underflow. + #[fail(display = "Math operator caused an underflow")] + MathUnderflow, + /// Mismatching types #[fail( - display = "ArrayFilter subscript output was not RadonBoolean (was `{}`)", - value + display = "Mismatching types in {}. Expected: {}, found: {}", + method, expected, found )] - ArrayFilterWrongSubscript { value: String }, - /// Failed to parse a Value from a buffer + MismatchingTypes { + method: String, + expected: &'static str, + found: &'static str, + }, + /// There was a tie after applying the mode reducer #[fail( - display = "Failed to parse a Value from a buffer. Error message: {}", - description + display = "There was a tie after applying the mode reducer on values: `{:?}`", + values )] - BufferIsNotValue { description: String }, + ModeTie { values: RadonArray, max_count: u16 }, /// No operator found in compound call #[fail(display = "No operator found in compound call")] NoOperatorInCompoundCall, @@ -83,24 +215,91 @@ pub enum RadError { /// The given operator code is not a valid natural number #[fail(display = "Operator code `{}` is not a valid natural number", code)] NotNaturalOperator { code: i128 }, + /// Failed to convert string to bool + #[fail( + display = "Failed to convert string to bool with error message: {}", + message + )] + ParseBool { message: String }, + /// Failed to convert string to float + #[fail( + display = "Failed to convert string to float with error message: {}", + message + )] + ParseFloat { message: String }, + /// Failed to convert string to int + #[fail( + display = "Failed to convert string to int with error message: {}", + message + )] + ParseInt { message: String }, + /// The request contains too many sources. + #[fail(display = "The request contains too many sources")] + RequestTooManySources, + /// Timeout during retrieval phase + #[fail(display = "Timeout during retrieval phase")] + RetrieveTimeout, /// The parsed value was expected to be a script but is not even an Array #[fail( display = "The parsed value was expected to be a script but is not even an Array (it was a `{}`)", input_type )] ScriptNotArray { input_type: String }, - /// The given operator code is unknown - #[fail(display = "Operator code `{}` is unknown", code)] - UnknownOperator { code: i128 }, + /// The script contains too many calls. + #[fail(display = "The script contains too many calls")] + ScriptTooManyCalls, + /// At least one of the source scripts is not a valid CBOR-encoded value. + #[fail(display = "At least one of the source scripts is not a valid CBOR-encoded value")] + SourceScriptNotCBOR, + /// The CBOR value decoded from a source script is not an Array. + #[fail(display = "The CBOR value decoded from a source script is not an Array")] + SourceScriptNotArray, + /// The Array value decoded form a source script is not a valid RADON script. + #[fail(display = "The Array value decoded form a source script is not a valid RADON script")] + SourceScriptNotRADON, + /// Error while executing subscript + #[fail( + display = "`{}::{}()`: Error in subscript: {}", + input_type, operator, inner + )] + Subscript { + input_type: String, + operator: String, + inner: Box, + }, + /// An unknown error. Something went really bad! + #[fail(display = "Unknown error")] + Unknown, /// The given filter code is unknown #[fail(display = "Filter code `{}` is unknown", code)] UnknownFilter { code: i128 }, + /// The given operator code is unknown + #[fail(display = "Operator code `{}` is unknown", code)] + UnknownOperator { code: i128 }, /// The given reducer code is unknown #[fail(display = "Reducer code `{}` is unknown", code)] UnknownReducer { code: i128 }, /// The given retrieval code is unknown #[fail(display = "Retrieval code is unknown")] - UnknownRetrieval, + UnknownRetrieval, + /// The given encoding schema is not implemented + #[fail( + display = "Encoding schema #{} not implemented", + schema + )] + UnsupportedEncodingSchema { schema: usize }, + /// The given filter is not implemented for the type of the input Array + #[fail( + display = "Filter `{}` is not implemented for Array `{:?}`", + filter, array + )] + UnsupportedFilter { array: RadonArray, filter: String }, + /// This filter cannot be used in aggregation or tally stage + #[fail( + display = "Filter {} cannot be used in aggregation or tally stage", + operator + )] + UnsupportedFilterInAT { operator: u8 }, /// The given hash function is not implemented #[fail(display = "Hash function `{}` is not implemented", function)] UnsupportedHashFunction { function: String }, @@ -114,51 +313,37 @@ pub enum RadError { operator: String, args: Option>, }, - /// The given reducer is not implemented for the type of the input Array - #[fail( - display = "Reducer `{}` is not implemented for Array `{:?}`", - reducer, array - )] - UnsupportedReducer { array: RadonArray, reducer: String }, - /// The given filter is not implemented for the type of the input Array - #[fail( - display = "Filter `{}` is not implemented for Array `{:?}`", - filter, array - )] - UnsupportedFilter { array: RadonArray, filter: String }, - /// The sort operator is not implemented for non-string arrays - #[fail(display = "ArraySort is not supported for RadonArray `{:?}`", array)] - UnsupportedSortOp { array: RadonArray }, + /// This operator cannot be used in tally stage + #[fail(display = "Operator {} cannot be used in tally stage", operator)] + UnsupportedOperatorInTally { operator: RadonOpCodes }, /// The operator is not implemented for non-homogeneous arrays #[fail( display = "`{}` is not supported for RadonArray with non homogeneous types", operator )] UnsupportedOpNonHomogeneous { operator: String }, - /// This operator cannot be used in tally stage - #[fail(display = "Operator {} cannot be used in tally stage", operator)] - UnsupportedOperatorInTally { operator: RadonOpCodes }, - /// This filter cannot be used in aggregation or tally stage + /// The given reducer is not implemented for the type of the input Array #[fail( - display = "Filter {} cannot be used in aggregation or tally stage", - operator + display = "Reducer `{}` is not implemented for Array `{:?}`", + reducer, array )] - UnsupportedFilterInAT { operator: u8 }, + UnsupportedReducer { array: RadonArray, reducer: String }, /// This reducer cannot be used in aggregation or tally stage #[fail( display = "Reducer {} cannot be used in aggregation or tally stage", operator )] UnsupportedReducerInAT { operator: u8 }, - /// There was a tie after applying the mode reducer - #[fail( - display = "There was a tie after applying the mode reducer on values: `{:?}`", - values - )] - ModeTie { values: RadonArray, max_count: u16 }, - /// Tried to apply mod reducer on an empty array - #[fail(display = "Tried to apply mode reducer on an empty array")] - EmptyArray, + /// The sort operator is not implemented for non-string arrays + #[fail(display = "ArraySort is not supported for RadonArray `{:?}`", array)] + UnsupportedSortOp { array: RadonArray }, + /// Error while parsing retrieval URL + #[fail(display = "URL parse error: {}: url={:?}", inner, url)] + UrlParseError { + #[cause] + inner: url::ParseError, + url: String, + }, /// The given arguments are not valid for the given operator #[fail( display = "Wrong `{}::{}()` arguments: `{:?}`", @@ -169,175 +354,75 @@ pub enum RadError { operator: String, args: Vec, }, - /// The HTTP response was an error code - #[fail(display = "HTTP GET response was an HTTP error code: {}", status_code)] - HttpStatus { status_code: u16 }, - /// Failed to execute HTTP request - #[fail( - display = "Failed to execute HTTP GET request with error message: {}", - message - )] - HttpOther { message: String }, - /// Failed to convert string to float - #[fail( - display = "Failed to convert string to float with error message: {}", - message - )] - ParseFloat { message: String }, - /// Failed to convert string to int - #[fail( - display = "Failed to convert string to int with error message: {}", - message - )] - ParseInt { message: String }, - /// Failed to convert string to bool - #[fail( - display = "Failed to convert string to bool with error message: {}", - message - )] - ParseBool { message: String }, - /// Overflow error - #[fail(display = "Overflow error")] - Overflow, - /// Mismatching types - #[fail( - display = "Mismatching types in {}. Expected: {}, found: {}", - method, expected, found - )] - MismatchingTypes { - method: String, - expected: &'static str, - found: &'static str, - }, - /// Arrays to be reduced have different sizes - #[fail( - display = "Arrays to be reduced in {} have different sizes. {} != {}", - method, first, second - )] - DifferentSizeArrays { - method: String, - first: usize, - second: usize, - }, - /// Subscripts should be an array - #[fail(display = "Subscript should be an array but is: {:?}", value)] - BadSubscriptFormat { value: SerdeCborValue }, - /// Error while executing subscript - #[fail( - display = "`{}::{}()`: Error in subscript: {}", - input_type, operator, inner - )] - Subscript { - input_type: String, - operator: String, - inner: Box, - }, - /// Error while parsing retrieval URL - #[fail(display = "URL parse error: {}: url={:?}", inner, url)] - UrlParseError { - #[cause] - inner: url::ParseError, - url: String, - }, - /// Timeout during retrieval phase - #[fail(display = "Timeout during retrieval phase")] - RetrieveTimeout, - /// Invalid script - #[fail( - display = "CBOR value cannot be translated into a proper RADON script: {:?}", - value - )] - InvalidScript { value: SerdeCborValue }, - /// Failed to encode `RadonError` arguments - #[fail(display = "Failed to encode `RadonError` arguments `{}`", error_args)] - EncodeRadonErrorArguments { error_args: String }, - /// Alleged `RadonError` is actually not an instance of `cbor::value::Value::Array` - #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value` that was not `Array` (was actually `{}`)", - actual_type - )] - DecodeRadonErrorNotArray { actual_type: String }, - /// Alleged `RadonError` is actually an empty `cbor::value::Value::Array` - #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because the array was empty" - )] - DecodeRadonErrorEmptyArray, - /// Alleged `RadonError` contains an error code that is not `u8` + /// Failed to parse an object from a XML buffer #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its first element (the error code) was not `cbor::value::Value::U8` (was actually `{}`)", - actual_type + display = "Failed to parse an object from a XML buffer: {:?}", + description )] - DecodeRadonErrorBadCode { actual_type: String }, - /// Alleged `RadonError` contains an unknown error code + XmlParse { description: String }, + /// Failed to parse an object from a XML buffer by depth overflow + #[fail(display = "Failed to parse an object from a XML buffer: XML depth overflow")] + XmlParseOverflow, + + /// =============================================================================================================== + /// Errors that can be eventually wrapped into a RadonTypes::RadonError --------------------------------- + + /// Received insufficient commits + #[fail(display = "Insufficient commits received")] + InsufficientCommits, + + /// Received insufficient reveals + #[fail(display = "Insufficient reveals received")] + InsufficientQuorum, + + /// Insufficient consensus in tally precondition clause #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its first element (`{:?}`) did not match any known error code", - error_code + display = "Tally precondition clause failed because of insufficient consensus (achieved: {}, required: {})", + achieved, required )] - DecodeRadonErrorUnknownCode { error_code: u8 }, - /// Alleged `RadonError` does not have any arguments + InsufficientMajority { achieved: f64, required: f64 }, + + /// A majority of data sources could either be temporarily unresponsive or failing to report the requested data: #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its arguments are empty" + display = "Data sources seems to be temporarily unresponsive: subcode {:?}: args: {:?}", + subcode, + args )] - DecodeRadonErrorMissingArguments, - /// Alleged `RadonError` does not have the expected arguments + CircumstantialFailure { subcode: u8, args: Option> }, + + /// At least one data source is inconsistent when queried through multiple transports at once: + #[fail(display = "Inconsistent sources")] + InconsistentSources, + + /// Any one of the (multiple) Retrieve, Aggregate or Tally scripts were badly formated: #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its arguments (`{:?}`) were not compatible: {}", - arguments, message + display = "Malformed data request: {:?}", + subcode )] - DecodeRadonErrorWrongArguments { - arguments: Option, - message: String, - }, - /// Alleged `RadonError` has a `RadonTypes` argument which was wrongly serialized - // FIXME(#953): this error should not exist, but it is useful to detect problems with the - // current hacky implementation + MalformedDataRequest { subcode: u8 }, + + /// Values returned from data sources seem not to match the expected schema: #[fail( - display = "Failed to decode a `RadonError` from a `cbor::value::Value::Array` because its arguments (`{:?}`) were not compatible: {}", - arguments, message + display = "Malformed response from data sources: {:?}", + subcode )] - DecodeRadonErrorArgumentsRadonTypesFail { - arguments: Option>, - message: String, - }, - /// No commits received - #[fail(display = "Insufficient commits received")] - InsufficientCommits, - /// No reveals received - #[fail(display = "No reveals received")] - NoReveals, - /// Insufficient consensus in tally precondition clause + MalformedResponses { subcode: u8 }, + + /// Size of serialized tally result exceeds allowance: #[fail( - display = "Tally precondition clause failed because of insufficient consensus (achieved: {}, required: {})", - achieved, required + display = "Tally result size exceeded allowance (actual: {}, allowance: {})", + actual, allowance )] - InsufficientConsensus { achieved: f64, required: f64 }, - /// The request contains too many sources. - #[fail(display = "The request contains too many sources")] - RequestTooManySources, - /// The script contains too many calls. - #[fail(display = "The script contains too many calls")] - ScriptTooManyCalls, - /// At least one of the source scripts is not a valid CBOR-encoded value. - #[fail(display = "At least one of the source scripts is not a valid CBOR-encoded value")] - SourceScriptNotCBOR, - /// The CBOR value decoded from a source script is not an Array. - #[fail(display = "The CBOR value decoded from a source script is not an Array")] - SourceScriptNotArray, - /// The Array value decoded form a source script is not a valid RADON script. - #[fail(display = "The Array value decoded form a source script is not a valid RADON script")] - SourceScriptNotRADON, - /// Math operator caused an underflow. - #[fail(display = "Math operator caused an underflow")] - Underflow, - /// Tried to divide by zero. - #[fail(display = "Tried to divide by zero")] - DivisionByZero, - /// `RadError` cannot be converted to `RadonError` because the error code is not defined + OversizedTallyResult { actual: usize, allowance: usize }, + + /// `RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result #[fail( - display = "`RadError` cannot be converted to `RadonError` because the error code is not defined" + display = "`RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result. Inner: `{:?}`", + inner )] - EncodeRadonErrorUnknownCode, - /// Generic error during tally execution + UnhandledInterceptV2 { inner: Option> }, + + /// Legacy: Errors during tally execution #[fail( display = "Error during tally execution. Message: {:?}. Inner: `{:?}`", message, inner @@ -346,7 +431,8 @@ pub enum RadError { inner: Option>, message: Option, }, - /// `RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result + + /// Legacy: `RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result #[fail( display = "`RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result. Message: {:?}. Inner: `{:?}`", message, inner @@ -355,33 +441,9 @@ pub enum RadError { inner: Option>, message: Option, }, - /// `RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result - #[fail( - display = "`RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result. Inner: `{:?}`", - inner - )] - UnhandledInterceptV2 { inner: Option> }, - /// Invalid reveal serialization (malformed reveals are converted to this value) - #[fail(display = "The reveal was not serialized correctly")] - MalformedReveal, - /// Failed to encode reveal - #[fail(display = "Error when encoding reveal value")] - EncodeReveal, - /// Error while parsing HTTP header - #[fail( - display = "invalid HTTP header: {}. name={:?}, value={:?}", - error, name, value - )] - InvalidHttpHeader { - name: String, - value: String, - error: String, - }, - /// Source looks inconsistent when queried through multiple transports at once. - #[fail(display = "Source looks inconsistent when queried through multiple transports at once")] - InconsistentSource, } + impl RadError { pub fn try_from_cbor_array( serde_cbor_array: Vec, From 48af1922778fa511472a30928d5c44fb38587676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 11:04:06 +0100 Subject: [PATCH 38/46] chore(rad): add subcrate where to place implementation of RadError's legacy methods --- rad/src/error.rs | 245 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 234 insertions(+), 11 deletions(-) diff --git a/rad/src/error.rs b/rad/src/error.rs index 25d641653..28a2e327f 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -802,6 +802,229 @@ impl TryFrom> for RadonTypes { } } +/// This module was introduced for encapsulating the interim legacy logic before WIP-0028 is +/// introduced, for the sake of maintainability. +pub mod legacy { + use super::*; + + pub fn deserialize_args( + error_args: Option>, + ) -> Result { + let error_args = if let Some(x) = error_args { + SerdeCborValue::Array(x) + } else { + return Err(RadError::DecodeRadonErrorMissingArguments); + }; + + serde_cbor::value::from_value(error_args.clone()).map_err(|e| { + RadError::DecodeRadonErrorWrongArguments { + arguments: Some(error_args), + message: e.to_string(), + } + }) + } + + pub fn serialize_args( + args: T, + ) -> Result { + serde_cbor::value::to_value(&args).map_err(|_| RadError::EncodeRadonErrorArguments { + error_args: format!("{:?}", args), + }) + } + + impl RadError { + + pub fn try_into_error_code_before_wip0028(&self) -> Result { + Ok(match self { + RadError::ArrayIndexOutOfBounds { .. } => RadonErrors::ArrayIndexOutOfBounds, + RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue, + RadError::EncodeReveal => RadonErrors::EncodeReveals, + RadError::HttpStatus { .. } => RadonErrors::HttpErrors, + RadError::InsufficientCommits => RadonErrors::InsufficientCommits, + RadError::InsufficientMajority { .. } => RadonErrors::InsufficientMajority, + RadError::InsufficientQuorum => RadonErrors::InsufficientQuorum, + RadError::MalformedReveal => RadonErrors::MalformedReveals, + RadError::MapKeyNotFound { .. } => RadonErrors::MapKeyNotFound, + RadError::MathDivisionByZero => RadonErrors::MathDivisionByZero, + RadError::MathOverflow => RadonErrors::MathOverflow, + RadError::MathUnderflow => RadonErrors::MathUnderflow, + RadError::RequestTooManySources => RadonErrors::RequestTooManySources, + RadError::RetrieveTimeout => RadonErrors::RetrievalsTimeout, + RadError::ScriptTooManyCalls => RadonErrors::ScriptTooManyCalls, + RadError::SourceScriptNotCBOR => RadonErrors::SourceScriptNotCBOR, + RadError::SourceScriptNotArray => RadonErrors::SourceScriptNotArray, + RadError::SourceScriptNotRADON => RadonErrors::SourceScriptNotRADON, + RadError::TallyExecution { .. } => RadonErrors::CircumstantialFailure, + RadError::UnhandledIntercept { .. } | RadError::UnhandledInterceptV2 { .. } => { + RadonErrors::UnhandledIntercept + } + RadError::UnsupportedOperator { .. } => RadonErrors::UnsupportedOperator, + + // The `InconsistentSource` error was mapped here before WIP#0028 for the sake of backwards + // compatibility. Namely, to enable paranoid retrieval without having to immediately + // introduce a breaking change that may jeopardize oracle queries. The point of making + // the mapping here was to only affect actual witnessing and committing, but not the + // `try_data_request` function, which could rather use the `InconsistentSource` error for + // clarity. + RadError::Unknown | RadError::InconsistentSource => RadonErrors::Unknown, + + _ => return Err(RadError::EncodeRadonErrorUnknownCode), + }) + } + + pub fn try_into_cbor_array_before_wip0028(&self) -> Result, RadError> { + let kind = u8::from(self.try_into_error_code_before_wip0028()?); + let args = match self { + + RadError::ArrayIndexOutOfBounds { index } => Some(serialize_args((index,))?), + RadError::BufferIsNotValue { description } => Some(serialize_args((description,))?), + RadError::HttpStatus { status_code } => Some(serialize_args((status_code,))?), + RadError::MapKeyNotFound { key } => Some(serialize_args((key,))?), + + RadError::InsufficientMajority { achieved, required } => { + Some(serialize_args((achieved, required))?) + } + + RadError::TallyExecution { inner, message } => { + let message = match (inner, message) { + // Only serialize the message + (_, Some(message)) => message.clone(), + // But if there is no message, serialize the debug representation of inner + (Some(inner), None) => format!("inner: {:?}", inner), + // And if there is no inner, serialize this string + (None, None) => "inner: None".to_string(), + }; + Some(serialize_args((message,))?) + } + + RadError::UnsupportedOperator { + input_type, + operator, + args, + } => Some(serialize_args((input_type, operator, args))?), + + RadError::UnhandledIntercept { inner, message } => { + let message = match (inner, message) { + // Only serialize the message + (_, Some(message)) => message.clone(), + // But if there is no message, serialize the debug representation of inner + (Some(inner), None) => { + // Fix #1993 by emulating a bug from old versions of Rust (rust-lang/rust#83046) + if_rust_version::if_rust_version! { >= 1.53 { + format!("inner: {:?}", inner).replace('\'', "\\'") + } else { + format!("inner: {:?}", inner) + }} + } + // And if there is no inner, serialize this string + (None, None) => "inner: None".to_string(), + }; + Some(serialize_args((message,))?) + } + + _ => None, + }; + + let mut v = vec![SerdeCborValue::Integer(i128::from(kind))]; + + match args { + None => {} + Some(SerdeCborValue::Array(a)) => { + // Append arguments to resulting array. The format of the resulting array is: + // [kind, arg0, arg1, arg2, ...] + v.extend(a); + } + Some(value) => { + // This can only happen if `serialize_args` is called with a non-tuple argument + // For example: + // `serialize_args(x)` is invalid, it should be `serialize_args((x,))` + panic!("Args should be an array, is {:?}", value); + } + } + + Ok(v) + } + + pub fn try_into_radon_error_before_wip0028( + kind: RadonErrors, + error_args: Option> + ) -> Result, Self> { + Ok(RadonError::new(match kind { + + RadonErrors::InsufficientCommits => RadError::InsufficientCommits, + RadonErrors::InsufficientQuorum => RadError::InsufficientQuorum, + RadonErrors::InsufficientMajority => { + let (achieved, required) = deserialize_args(error_args)?; + RadError::InsufficientMajority { achieved, required } + } + RadonErrors::EncodeReveals => RadError::EncodeReveal, + RadonErrors::ArrayIndexOutOfBounds => { + let (index,) = deserialize_args(error_args)?; + RadError::ArrayIndexOutOfBounds { index } + } + RadonErrors::BufferIsNotValue => { + let (description,) = deserialize_args(error_args)?; + RadError::BufferIsNotValue { description } + } + RadonErrors::HttpErrors => { + let (status_code,) = deserialize_args(error_args)?; + RadError::HttpStatus { status_code } + } + RadonErrors::MapKeyNotFound => { + let (key,) = deserialize_args(error_args)?; + RadError::MapKeyNotFound { key } + } + RadonErrors::MathDivisionByZero => RadError::MathDivisionByZero, + RadonErrors::MathOverflow => RadError::MathOverflow, + RadonErrors::MathUnderflow => RadError::MathUnderflow, + RadonErrors::RequestTooManySources => RadError::RequestTooManySources, + RadonErrors::RetrievalsTimeout => RadError::RetrieveTimeout, + RadonErrors::ScriptTooManyCalls => RadError::ScriptTooManyCalls, + RadonErrors::SourceScriptNotCBOR => RadError::SourceScriptNotCBOR, + RadonErrors::SourceScriptNotArray => RadError::SourceScriptNotArray, + RadonErrors::SourceScriptNotRADON => RadError::SourceScriptNotRADON, + RadonErrors::TallyExecution => { + let (message,) = deserialize_args(error_args)?; + RadError::TallyExecution { + inner: None, + message: Some(message), + } + } + RadonErrors::MalformedReveals => RadError::MalformedReveal, + RadonErrors::UnsupportedOperator => { + let (input_type, operator, args) = deserialize_args(error_args)?; + RadError::UnsupportedOperator { + input_type, + operator, + args, + } + } + RadonErrors::UnhandledIntercept => { + if error_args.is_none() { + RadError::UnhandledInterceptV2 { inner: None } + } else { + let (message,) = deserialize_args(error_args)?; + RadError::UnhandledIntercept { + inner: None, + message: Some(message), + } + } + } + // The only case where a Bridge RadonError could be included in the protocol is that + // if a witness node report as a reveal, and in that case it would be considered + // as a MalformedReveal + RadonErrors::BridgeMalformedRequest + | RadonErrors::BridgePoorIncentives + | RadonErrors::BridgeOversizedResult => RadError::MalformedReveal, + + _ => RadError::Unknown + })) + } + + } +} + + #[cfg(test)] mod tests { use num_enum::TryFromPrimitive; @@ -816,8 +1039,8 @@ mod tests { operator: "IntegerAdd".to_string(), args: Some(vec![SerdeCborValue::Integer(1)]), }, - RadonErrors::HTTPError => RadError::HttpStatus { status_code: 404 }, - RadonErrors::InsufficientConsensus => RadError::InsufficientConsensus { + RadonErrors::HttpErrors => RadError::HttpStatus { status_code: 404 }, + RadonErrors::InsufficientMajority => RadError::InsufficientMajority { achieved: 49.0, required: 51.0, }, @@ -829,6 +1052,9 @@ mod tests { RadonErrors::MapKeyNotFound => RadError::MapKeyNotFound { key: String::from("value"), }, + RadonErrors::JsonPathNotFound => RadError::JsonPathNotFound { + path: String::from("*.*") + }, RadonErrors::UnhandledIntercept => RadError::UnhandledIntercept { inner: None, message: Some("Only the message field is serialized".to_string()), @@ -870,7 +1096,7 @@ mod tests { for radon_errors in all_radon_errors() { // Try to convert RadonErrors to RadError with no arguments let maybe_rad_error = - RadError::try_from_kind_and_cbor_args(radon_errors, None).map(|r| r.into_inner()); + RadError::try_into_radon_error_before_wip0028(radon_errors, None).map(|r| r.into_inner()); let rad_error = match maybe_rad_error { Ok(x) => { // Good @@ -884,11 +1110,7 @@ mod tests { }; // Now try the inverse: convert from RadError to RadonErrors - let again_radon_errors = rad_error.try_into_error_code(); - match again_radon_errors { - Ok(x) => assert_eq!(x, radon_errors), - Err(e) => panic!("RadonErrors::{:?}: {}", radon_errors, e), - } + assert_eq!(rad_error.into_error_code(), radon_errors); } } @@ -897,7 +1119,7 @@ mod tests { for radon_errors in all_radon_errors() { // Try to convert RadonErrors to RadError with no arguments let maybe_rad_error = - RadError::try_from_kind_and_cbor_args(radon_errors, None).map(|r| r.into_inner()); + RadError::try_into_radon_error_before_wip0028(radon_errors, None).map(|r| r.into_inner()); let rad_error = match maybe_rad_error { Ok(x) => { // Good @@ -911,7 +1133,7 @@ mod tests { }; // Now try to serialize the resulting rad_error - let serde_cbor_array = match rad_error.try_into_cbor_array() { + let serde_cbor_array = match rad_error.try_into_cbor_array_before_wip0028() { Ok(x) => x, Err(e) => panic!("RadonErrors::{:?}: {}", radon_errors, e), }; @@ -921,6 +1143,7 @@ mod tests { assert_eq!(serde_cbor_array[0], Value::Integer(error_code.into())); // Deserialize the result and compare + println!("{:?}", serde_cbor_array[0]); let deserialized_rad_error = RadError::try_from_cbor_array(serde_cbor_array).map(|r| r.into_inner()); @@ -948,7 +1171,7 @@ mod tests { }; // Now try to serialize the resulting rad_error - let serde_cbor_array = rad_error.try_into_cbor_array().unwrap(); + let serde_cbor_array = rad_error.try_into_cbor_array_before_wip0028().unwrap(); // The first element of the serialized CBOR array is the error code // the rest are arguments let error_code = u8::from(RadonErrors::UnhandledIntercept); From e5625dd13066ada98346ff887a66925e3fcb36fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 11:18:50 +0100 Subject: [PATCH 39/46] feat(rad): add active_wips control logic to the encoding of RadonTypes::RadonError --- data_structures/src/chain/tapi.rs | 4 + data_structures/src/data_request.rs | 2 +- data_structures/src/radon_error.rs | 14 ++- data_structures/src/radon_report.rs | 28 +++--- node/src/actors/chain_manager/mining.rs | 2 +- rad/src/conditions.rs | 6 +- rad/src/error.rs | 20 ++--- rad/src/lib.rs | 2 +- rad/src/types/mod.rs | 112 ++++++++++++++---------- validations/src/tests/mod.rs | 62 ++++++------- 10 files changed, 141 insertions(+), 111 deletions(-) diff --git a/data_structures/src/chain/tapi.rs b/data_structures/src/chain/tapi.rs index ec000ec1e..95d7d23ee 100644 --- a/data_structures/src/chain/tapi.rs +++ b/data_structures/src/chain/tapi.rs @@ -555,6 +555,10 @@ impl ActiveWips { self.wip_active("WIP0027") } + pub fn wip_guidiaz_extended_radon_errors(&self) -> bool { + self.wip_active("WIP0028") + } + /// Convenience method for inserting WIPs. pub fn insert_wip(&mut self, wip: &str, activation_epoch: Epoch) { self.active_wips.insert(String::from(wip), activation_epoch); diff --git a/data_structures/src/data_request.rs b/data_structures/src/data_request.rs index fc3823037..711208165 100644 --- a/data_structures/src/data_request.rs +++ b/data_structures/src/data_request.rs @@ -630,7 +630,7 @@ where // Check if we need to reward revealers let any_honest_revealers = if is_after_second_hard_fork { let honests_count = commits_count - liars_count - errors_count - non_reveals_count; - + honests_count > 0 } else { reveals_count > 0 diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs index 78171cb4d..ab9771f9b 100644 --- a/data_structures/src/radon_error.rs +++ b/data_structures/src/radon_error.rs @@ -163,11 +163,9 @@ impl Default for RadonErrors { /// `RadonReport`. pub trait ErrorLike: Clone + Fail { /// Encode the error as an array of `SerdeCborValue` - fn encode_cbor_array(&self) -> Result, failure::Error>; + fn encode_cbor_array(&self, active_wips: &Option) -> Result, failure::Error>; /// Decode the error from an array of `SerdeCborValue` - fn decode_cbor_array( - serde_cbor_array: Vec, - ) -> Result, failure::Error>; + fn decode_cbor_array(serde_cbor_array: Vec) -> Result, failure::Error>; } /// This structure is aimed to be the error type for the `result` field of `witnet_data_structures::radon_report::Report`. @@ -192,10 +190,10 @@ where /// Encode `RadonError` as tagged CBOR value with tag 39. /// Returns the result as `CborValue`. - pub fn encode_tagged_value(&self) -> Result { + pub fn encode_tagged_value(&self, active_wips: &Option) -> Result { let values: Vec = self .inner - .encode_cbor_array()? + .encode_cbor_array(active_wips)? .into_iter() .map(|scv| { // FIXME(#953): remove this conversion @@ -211,9 +209,9 @@ where /// Encode `RadonErorr` as tagged CBOR value with tag 39. /// Returns the result as bytes. - pub fn encode_tagged_bytes(&self) -> Result, failure::Error> { + pub fn encode_tagged_bytes(&self, active_wips: &Option) -> Result, failure::Error> { let mut encoder = GenericEncoder::new(Cursor::new(Vec::new())); - encoder.value(&self.encode_tagged_value()?)?; + encoder.value(&self.encode_tagged_value(active_wips)?)?; Ok(encoder.into_inner().into_writer().into_inner()) } diff --git a/data_structures/src/radon_report.rs b/data_structures/src/radon_report.rs index 8ff2bbd60..b26a0c7fe 100644 --- a/data_structures/src/radon_report.rs +++ b/data_structures/src/radon_report.rs @@ -36,7 +36,7 @@ where /// Factory for constructing a `RadonReport` from the `Result` of something that could be /// `TypeLike` or `ErrorLike` plus a `ReportContext`. pub fn from_result(result: Result, context: &ReportContext) -> Self { - let intercepted = RT::intercept(result); + let intercepted = RT::intercept(result, &context.active_wips); RadonReport { context: context.clone(), partial_results: None, @@ -51,7 +51,10 @@ where partial_results: Vec>, context: &ReportContext, ) -> Self { - let intercepted: Vec = partial_results.into_iter().map(RT::intercept).collect(); + let intercepted: Vec = partial_results + .into_iter() + .map(|r| RT::intercept(r, &context.active_wips)) + .collect(); let result = (*intercepted .last() .expect("Partial result vectors must always contain at least 1 item")) @@ -80,7 +83,7 @@ where type Error = RT::Error; fn try_from(report: &RadonReport) -> Result { - report.result.encode() + report.result.encode(&report.context.active_wips) } } @@ -91,12 +94,13 @@ pub trait TypeLike: Clone + Sized { type Error: ErrorLike; /// Serialize the `TypeLike` as a `Vec`. - fn encode(&self) -> Result, Self::Error>; - + fn encode(&self, _active_wips: &Option) -> Result, Self::Error>; + fn encode_legacy(&self) -> Result, Self::Error>; + /// Eases interception of RADON errors (errors that we want to commit, reveal and tally) so /// they can be handled as valid `RadonTypes::RadonError` values, which are subject to /// commitment, revealing, tallying, etc. - fn intercept(result: Result) -> Self; + fn intercept(result: Result, _active_wips: &Option) -> Self; } /// A generic structure for bubbling up any kind of metadata that may be generated during the @@ -348,11 +352,15 @@ mod tests { impl TypeLike for DummyType { type Error = DummyError; - fn encode(&self) -> Result, Self::Error> { + fn encode(&self, _active_wips: &Option) -> Result, Self::Error> { + unimplemented!() + } + + fn encode_legacy(&self) -> Result, Self::Error> { unimplemented!() } - fn intercept(_result: Result) -> Self { + fn intercept(_result: Result, _active_wips: &Option) -> Self { unimplemented!() } } @@ -366,7 +374,7 @@ mod tests { // Satisfy the trait bound `Dummy: radon_error::ErrorLike` required by `radon_error::RadonError` impl ErrorLike for DummyError { - fn encode_cbor_array(&self) -> Result, failure::Error> { + fn encode_cbor_array(&self, _active_wips: &Option) -> Result, failure::Error> { let kind = u8::from(RadonErrors::SourceScriptNotCBOR); let arg0 = 2; @@ -394,7 +402,7 @@ mod tests { fn test_encode_not_cbor() { let error = RadonError::new(DummyError); - let encoded: Vec = error.encode_tagged_bytes().unwrap(); + let encoded: Vec = error.encode_tagged_bytes(&None).unwrap(); let expected = vec![216, 39, 130, 1, 2]; assert_eq!(encoded, expected); diff --git a/node/src/actors/chain_manager/mining.rs b/node/src/actors/chain_manager/mining.rs index 231bdfb9c..6123caa4f 100644 --- a/node/src/actors/chain_manager/mining.rs +++ b/node/src/actors/chain_manager/mining.rs @@ -603,7 +603,7 @@ impl ChainManager { Err(e) => { if active_wips.wip0026() { log::warn!("Couldn't encode reveal value to bytes, will commit RadError::EncodeReveal instead: {}", e); - let reveal_bytes: Vec = RadonTypes::RadonError(RadonError::try_from(RadError::EncodeReveal).unwrap()).encode().unwrap(); + let reveal_bytes: Vec = RadonTypes::RadonError(RadonError::try_from(RadError::EncodeReveal).unwrap()).encode(&Some(active_wips)).unwrap(); actix::fut::ok((reveal_bytes, vrf_proof_dr, collateral)) } else { log::error!("Couldn't encode reveal value to bytes: {}", e); diff --git a/rad/src/conditions.rs b/rad/src/conditions.rs index 181888b5b..5f23e4d62 100644 --- a/rad/src/conditions.rs +++ b/rad/src/conditions.rs @@ -55,7 +55,7 @@ pub fn evaluate_tally_precondition_clause( } let error_type_discriminant = - RadonTypes::RadonError(RadonError::try_from(RadError::default()).unwrap()).discriminant(); + RadonTypes::RadonError(RadonError::new(RadError::default())).discriminant(); // Count how many times is each RADON type featured in `reveals`, but count `RadonError` items // separately as they need to be handled differently. @@ -72,6 +72,7 @@ pub fn evaluate_tally_precondition_clause( } let mut num_commits_f = f64::from(u32::try_from(num_commits).unwrap()); + // Compute ratio of type consensus amongst reveals (percentage of reveals that have same type // as the frequent type). let achieved_consensus = f64::from(counter.max_val) / num_commits_f; @@ -190,8 +191,7 @@ pub fn evaluate_tally_postcondition_clause( ) -> RadonReport { if let Stage::Tally(metadata) = report.context.stage.clone() { let error_type_discriminant = - RadonTypes::RadonError(RadonError::try_from(RadError::default()).unwrap()) - .discriminant(); + RadonTypes::RadonError(RadonError::new(RadError::default())).discriminant(); // If the result is already a RadonError, return that error. // The result can be a RadonError in these scenarios: // * There is insufficient consensus before running the tally script diff --git a/rad/src/error.rs b/rad/src/error.rs index 28a2e327f..a0a64c341 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -697,18 +697,22 @@ impl RadError { other => other, } } + } /// Satisfy the `ErrorLike` trait that ensures generic compatibility of `witnet_rad` and /// `witnet_data_structures`. impl ErrorLike for RadError { - fn encode_cbor_array(&self) -> Result, failure::Error> { - self.try_into_cbor_array().map_err(Into::into) + fn encode_cbor_array(&self, active_wips: &Option) -> Result, failure::Error> { + if let Some(active_wips) = active_wips { + if active_wips.wip_guidiaz_extended_radon_errors() { + return self.try_into_cbor_array().map_err(Into::into); + } + } + self.try_into_cbor_array_before_wip0028().map_err(Into::into) } - fn decode_cbor_array( - serde_cbor_array: Vec, - ) -> Result, failure::Error> { + fn decode_cbor_array(serde_cbor_array: Vec) -> Result, failure::Error> { Self::try_from_cbor_array(serde_cbor_array).map_err(Into::into) } } @@ -777,11 +781,7 @@ impl TryFrom for RadonError { /// This is the main logic for intercepting `RadError` items and converting them into /// `RadonError` so that they can be committed, revealed, tallied, etc. fn try_from(rad_error: RadError) -> Result { - // Assume that there exists a conversion if try_into_error_code returns Ok - match rad_error.try_into_error_code() { - Ok(_) => Ok(RadonError::new(rad_error)), - Err(_) => Err(rad_error), - } + Ok(RadonError::new(rad_error)) } } diff --git a/rad/src/lib.rs b/rad/src/lib.rs index c62706c34..453acde01 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -1545,7 +1545,7 @@ mod tests { let kind = RadonErrors::try_from(0).unwrap(); let err = RadonTypes::from(RadError::try_from_kind_and_cbor_args(kind, None).unwrap()); // Ensure they encoded differently (errors are tagged using `39` as CBOR tag) - assert_ne!(int.encode(), err.encode()); + assert_ne!(int.encode_legacy(), err.encode_legacy()); // And they are not equal in runtime either assert_ne!(int, err); } diff --git a/rad/src/types/mod.rs b/rad/src/types/mod.rs index 0a5f94824..81e94ddf5 100644 --- a/rad/src/types/mod.rs +++ b/rad/src/types/mod.rs @@ -11,7 +11,7 @@ use serde_json::Value as JsonValue; use witnet_crypto::hash::calculate_sha256; use witnet_data_structures::{ chain::{tapi::ActiveWips, Hash}, - radon_error::{try_from_cbor_value_for_serde_cbor_value, RadonError}, + radon_error::{try_from_cbor_value_for_serde_cbor_value, RadonError, RadonErrors}, radon_report::{RadonReport, ReportContext, TypeLike}, }; @@ -55,7 +55,7 @@ pub enum RadonTypes { impl RadonTypes { pub fn hash(self) -> Result { - self.encode() + self.encode_legacy() .map(|vector: Vec| calculate_sha256(&vector)) .map(Hash::from) .map_err(|_| RadError::Hash) @@ -139,24 +139,74 @@ impl TypeLike for RadonTypes { type Error = RadError; // FIXME(953): Unify all CBOR libraries - fn encode(&self) -> Result, Self::Error> { - Vec::::try_from((*self).clone()) + fn encode_legacy(&self) -> Result, Self::Error> { + let type_name = self.radon_type_name(); + let value: Value = self.clone().try_into()?; + to_vec(&value).map_err(|_| RadError::Encode { + from: type_name, + to: "Vec", + }) + } + + // FIXME(953): Unify all CBOR libraries + fn encode(&self, active_wips: &Option) -> Result, Self::Error> { + let type_name = self.radon_type_name(); + match self { + RadonTypes::RadonError(radon_error) => { + radon_error + .encode_tagged_bytes(active_wips) + .map_err(|_| RadError::Encode { + from: type_name, + to: "Vec", + }) + } + _ => { + let value: Value = self.clone().try_into()?; + to_vec(&value).map_err(|_| RadError::Encode { + from: type_name, + to: "Vec", + }) + } + } } /// Eases interception of RADON errors (errors that we want to commit, reveal and tally) so /// they can be handled as valid `RadonTypes::RadonError` values, which are subject to /// commitment, revealing, tallying, etc. - fn intercept(result: Result) -> Self { + fn intercept(result: Result, active_wips: &Option) -> Self { + let active_wip0028 = active_wips.as_ref() + .map(ActiveWips::wip_guidiaz_extended_radon_errors) + .unwrap_or_default(); match result { Err(rad_error) => { - RadonTypes::RadonError(RadonError::try_from(rad_error).unwrap_or_else(|error| { - let unhandled_rad_error = RadError::UnhandledIntercept { - inner: Some(Box::new(error)), - message: None, - }; - log::warn!("{}", unhandled_rad_error); - RadonError::new(unhandled_rad_error) - })) + if active_wip0028 { + RadonTypes::RadonError(match rad_error.into_error_code() { + RadonErrors::UnhandledIntercept => { + log::warn!("Unhandled RadError: {}", rad_error); + RadonError::new(RadError::UnhandledIntercept { + inner: Some(Box::new(rad_error)), + message: None, + }) + } + _ => { + RadonError::new(rad_error) + } + }) + } else { + RadonTypes::RadonError(match rad_error.try_into_error_code_before_wip0028() { + Err(error) => { + let unhandled_rad_error = RadError::UnhandledIntercept { + inner: Some(Box::new(error)), + message: None, + }; + log::warn!("{}", unhandled_rad_error); + RadonError::new(unhandled_rad_error) + } + _ => { + RadonError::new(rad_error) + } + }) + } } Ok(x) => x, } @@ -214,8 +264,8 @@ impl PartialEq for RadonTypes { return false; } - let vec1 = self.encode(); - let vec2 = other.encode(); + let vec1 = self.encode_legacy(); + let vec2 = other.encode_legacy(); vec1 == vec2 } @@ -224,7 +274,7 @@ impl PartialEq for RadonTypes { impl std::hash::Hash for RadonTypes { // FIXME(953): Unify all CBOR libraries fn hash(&self, state: &mut H) { - match self.encode() { + match self.encode_legacy() { Ok(vec) => vec.hash(state), Err(e) => { let error_vec = e.to_string(); @@ -358,36 +408,6 @@ impl TryFrom<&[u8]> for RadonTypes { } } -/// Allow CBOR encoding of any variant of `RadonTypes`. -impl TryFrom for Vec { - type Error = RadError; - - // FIXME(953): Unify all CBOR libraries - fn try_from( - radon_types: RadonTypes, - ) -> Result, as TryFrom>::Error> { - let type_name = RadonTypes::radon_type_name(&radon_types); - - match radon_types { - RadonTypes::RadonError(radon_error) => { - radon_error - .encode_tagged_bytes() - .map_err(|_| RadError::Encode { - from: type_name, - to: "Vec", - }) - } - _ => { - let value: Value = radon_types.try_into()?; - to_vec(&value).map_err(|_| RadError::Encode { - from: type_name, - to: "Vec", - }) - } - } - } -} - // FIXME(953): migrate everything to using `cbor-codec` or wait for `serde_cbor` to support CBOR tags. /// Allow decoding RADON types also from `Value` structures coming from the `cbor-codec` crate. /// Take into account the difference between `cbor::value::Value` and `serde_cbor::Value`. diff --git a/validations/src/tests/mod.rs b/validations/src/tests/mod.rs index b14bb19c3..909ed16a6 100644 --- a/validations/src/tests/mod.rs +++ b/validations/src/tests/mod.rs @@ -4730,7 +4730,7 @@ fn dr_pool_with_dr_in_tally_stage_generic( get_rewarded_and_slashed(reveals_count, liars, errors, commits); // TODO: here liars_count must be equal to reveals_count if the result of the data request is - // "InsufficientConsensus" + // "InsufficientMajority" // Calculate tally change let change = calculate_tally_change( commits_count, @@ -4847,7 +4847,7 @@ fn dr_pool_with_dr_in_tally_stage_with_errors( &ReportContext::default(), ) .result - .encode() + .encode_legacy() .unwrap(); dr_pool_with_dr_in_tally_stage_generic( @@ -5064,7 +5064,7 @@ fn tally_valid_1_reveal_5_commits() { let tally_value = RadonReport::from_result( Ok(RadonTypes::from( - RadonError::try_from(RadError::InsufficientConsensus { + RadonError::try_from(RadError::InsufficientMajority { achieved: 0.2, required: 0.51, }) @@ -5072,7 +5072,7 @@ fn tally_valid_1_reveal_5_commits() { )), &ReportContext::default(), ); - let tally_bytes = tally_value.into_inner().encode().unwrap(); + let tally_bytes = tally_value.into_inner().encode_legacy().unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -5255,7 +5255,7 @@ fn tally_valid_1_reveal_5_commits_invalid_value() { let tally_value = RadonReport::from_result( Ok(RadonTypes::from( - RadonError::try_from(RadError::InsufficientConsensus { + RadonError::try_from(RadError::InsufficientMajority { achieved: 0.2, required: 0.51, }) @@ -5263,7 +5263,7 @@ fn tally_valid_1_reveal_5_commits_invalid_value() { )), &ReportContext::default(), ); - let tally_bytes = tally_value.into_inner().encode().unwrap(); + let tally_bytes = tally_value.into_inner().encode_legacy().unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -5327,7 +5327,7 @@ fn tally_valid_1_reveal_5_commits_with_absurd_timelock() { let tally_value = RadonReport::from_result( Ok(RadonTypes::from( - RadonError::try_from(RadError::InsufficientConsensus { + RadonError::try_from(RadError::InsufficientMajority { achieved: 0.2, required: 0.51, }) @@ -5335,7 +5335,7 @@ fn tally_valid_1_reveal_5_commits_with_absurd_timelock() { )), &ReportContext::default(), ); - let tally_bytes = tally_value.into_inner().encode().unwrap(); + let tally_bytes = tally_value.into_inner().encode_legacy().unwrap(); let vt0 = ValueTransferOutput { time_lock: u64::MAX, @@ -6320,8 +6320,8 @@ fn create_tally_validation_dr_liar() { 3, 3, 1, - reveal_value.result.encode().unwrap(), - liar_value.result.encode().unwrap(), + reveal_value.result.encode_legacy().unwrap(), + liar_value.result.encode_legacy().unwrap(), active_wips.clone(), ); @@ -6403,8 +6403,8 @@ fn create_tally_validation_5_reveals_1_liar_1_error() { 5, 1, 1, - reveal_value.result.encode().unwrap(), - liar_value.result.encode().unwrap(), + reveal_value.result.encode_legacy().unwrap(), + liar_value.result.encode_legacy().unwrap(), active_wips.clone(), ); @@ -6493,7 +6493,7 @@ fn create_tally_validation_4_commits_2_reveals() { 4, 2, 0, - reveal_value.result.encode().unwrap(), + reveal_value.result.encode_legacy().unwrap(), vec![], active_wips.clone(), ); @@ -6559,7 +6559,7 @@ fn tally_valid_zero_commits() { let script = RADTally::default(); let report = construct_report_from_clause_result(clause_result, &script, 0, &active_wips); let report = evaluate_tally_postcondition_clause(report, min_consensus, 0); - let tally_value = report.result.encode().unwrap(); + let tally_value = report.result.encode_legacy().unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, pkh: dr_pkh, @@ -6624,7 +6624,7 @@ fn tally_invalid_zero_commits() { let script = RADTally::default(); let report = construct_report_from_clause_result(clause_result, &script, 0, &active_wips); let report = evaluate_tally_postcondition_clause(report, min_consensus, 0); - let tally_value = report.result.encode().unwrap(); + let tally_value = report.result.encode_legacy().unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, pkh: PublicKeyHash::default(), @@ -6670,13 +6670,13 @@ fn tally_valid_zero_reveals() { // You get your collateral back assert_eq!(reward, DEFAULT_COLLATERAL); - // Tally value: NoReveals commits Error + // Tally value: InsufficientQuorum commits Error let min_consensus = 0.51; let clause_result = evaluate_tally_precondition_clause(vec![], min_consensus, 5, &active_wips); let script = RADTally::default(); let report = construct_report_from_clause_result(clause_result, &script, 0, &active_wips); let report = evaluate_tally_postcondition_clause(report, min_consensus, 5); - let tally_value = report.result.encode().unwrap(); + let tally_value = report.result.encode_legacy().unwrap(); assert_eq!(reward, dr_output.collateral); let vt1 = ValueTransferOutput { @@ -6738,7 +6738,7 @@ fn create_tally_validation_zero_reveals() { // There were no reveals, you get your collateral back assert_eq!(reward, DEFAULT_COLLATERAL); - // Tally value: NoReveals commits Error + // Tally value: InsufficientQuorum commits Error let min_consensus = 0.51; let clause_result = evaluate_tally_precondition_clause(vec![], min_consensus, 5, &active_wips); let script = RADTally::default(); @@ -6785,7 +6785,7 @@ fn create_tally_validation_zero_reveals_zero_collateral() { // No rewards, you just get your collateral back assert_eq!(reward, DEFAULT_COLLATERAL); - // Tally value: NoReveals commits Error + // Tally value: InsufficientQuorum commits Error let min_consensus = 0.51; let clause_result = evaluate_tally_precondition_clause(vec![], min_consensus, 5, &active_wips); let script = RADTally::default(); @@ -7714,14 +7714,14 @@ fn tally_valid_rng() { .unwrap(), )), ]; - let reveals = reveals.into_iter().map(|x| x.encode().unwrap()).collect(); + let reveals = reveals.into_iter().map(|x| x.encode_legacy().unwrap()).collect(); let (pkhs, _dr_pkh, dr_pointer, dr_pool) = generic_tally_test_rng(4, reveals); let reward = DEFAULT_WITNESS_REWARD + DEFAULT_COLLATERAL; let tally_value = RadonTypes::from(RadonBytes::from( hex::decode("26dd11b019b78ed2aea221fd9d27a90031c6550b004e443b7e81f1a176235a26").unwrap(), )) - .encode() + .encode_legacy() .unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -7768,14 +7768,14 @@ fn tally_valid_rng_wrong_bytes_len() { RadonTypes::from(RadonBytes::from(hex::decode("4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce").unwrap())), RadonTypes::from(RadonBytes::from(hex::decode("4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8affffffffffffffffffffffff").unwrap())), ]; - let reveals = reveals.into_iter().map(|x| x.encode().unwrap()).collect(); + let reveals = reveals.into_iter().map(|x| x.encode_legacy().unwrap()).collect(); let (pkhs, _dr_pkh, dr_pointer, dr_pool) = generic_tally_test_rng(4, reveals); let reward = DEFAULT_WITNESS_REWARD + DEFAULT_COLLATERAL; let tally_value = RadonTypes::from(RadonBytes::from( hex::decode("8ccdac64f7d20bcaff0ac8306ce67d7dfe2623feaa5c9066aa6c91e4a1b0dddd").unwrap(), )) - .encode() + .encode_legacy() .unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -7837,7 +7837,7 @@ fn tally_valid_rng_one_error() { .unwrap(), ), ]; - let reveals = reveals.into_iter().map(|x| x.encode().unwrap()).collect(); + let reveals = reveals.into_iter().map(|x| x.encode_legacy().unwrap()).collect(); let (pkhs, dr_pkh, dr_pointer, dr_pool) = generic_tally_test_rng(4, reveals); let collateral = DEFAULT_COLLATERAL; let reward = collateral + DEFAULT_WITNESS_REWARD; @@ -7846,7 +7846,7 @@ fn tally_valid_rng_one_error() { let tally_value = RadonTypes::from(RadonBytes::from( hex::decode("d167af9c78f098e4b064f7f8c98a34f27b406a529565c1da9e832cdd52e332cb").unwrap(), )) - .encode() + .encode_legacy() .unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -7922,7 +7922,7 @@ fn tally_valid_rng_all_errors() { .unwrap(), ), ]; - let reveals = reveals.into_iter().map(|x| x.encode().unwrap()).collect(); + let reveals = reveals.into_iter().map(|x| x.encode_legacy().unwrap()).collect(); let (pkhs, _dr_pkh, dr_pointer, dr_pool) = generic_tally_test_rng(4, reveals); let collateral = DEFAULT_COLLATERAL; let reward = collateral + DEFAULT_WITNESS_REWARD; @@ -7934,7 +7934,7 @@ fn tally_valid_rng_all_errors() { }) .unwrap(), ) - .encode() + .encode_legacy() .unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -7990,7 +7990,7 @@ fn tally_valid_rng_one_invalid_type() { )), RadonTypes::from(RadonInteger::from(4)), ]; - let reveals = reveals.into_iter().map(|x| x.encode().unwrap()).collect(); + let reveals = reveals.into_iter().map(|x| x.encode_legacy().unwrap()).collect(); let (pkhs, dr_pkh, dr_pointer, dr_pool) = generic_tally_test_rng(4, reveals); let collateral = DEFAULT_COLLATERAL; let active_wips = current_active_wips(); @@ -8004,7 +8004,7 @@ fn tally_valid_rng_one_invalid_type() { let tally_value = RadonTypes::from(RadonBytes::from( hex::decode("d167af9c78f098e4b064f7f8c98a34f27b406a529565c1da9e832cdd52e332cb").unwrap(), )) - .encode() + .encode_legacy() .unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, @@ -8051,7 +8051,7 @@ fn tally_valid_rng_all_invalid_type() { RadonTypes::from(RadonInteger::from(3)), RadonTypes::from(RadonInteger::from(4)), ]; - let reveals = reveals.into_iter().map(|x| x.encode().unwrap()).collect(); + let reveals = reveals.into_iter().map(|x| x.encode_legacy().unwrap()).collect(); let (pkhs, _dr_pkh, dr_pointer, dr_pool) = generic_tally_test_rng(4, reveals); let collateral = DEFAULT_COLLATERAL; let reward = collateral + DEFAULT_WITNESS_REWARD; @@ -8059,7 +8059,7 @@ fn tally_valid_rng_all_invalid_type() { let tally_value = RadonTypes::from( RadonError::try_from(RadError::UnhandledInterceptV2 { inner: None }).unwrap(), ) - .encode() + .encode_legacy() .unwrap(); let vt0 = ValueTransferOutput { time_lock: 0, From f6a457495e8a1a30432badf6aef2cd3d860cc903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 11:45:10 +0100 Subject: [PATCH 40/46] refactor(rad): *::NoReveals -> *::InsufficientQuorum --- rad/src/conditions.rs | 2 +- rad/src/lib.rs | 4 ++-- validations/src/tests/tally_precondition.rs | 2 +- wallet/src/repository/wallet/mod.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rad/src/conditions.rs b/rad/src/conditions.rs index 5f23e4d62..eb5c93dd7 100644 --- a/rad/src/conditions.rs +++ b/rad/src/conditions.rs @@ -51,7 +51,7 @@ pub fn evaluate_tally_precondition_clause( } // Short-circuit if there were no reveals if reveals.is_empty() { - return Err(RadError::NoReveals); + return Err(RadError::InsufficientQuorum); } let error_type_discriminant = diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 453acde01..b0ad3dc2b 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -626,7 +626,7 @@ pub fn run_tally_with_context_report( create_radon_script_from_filters_and_reducer(filters, reducer, &active_wips)?; if radon_types_vec.is_empty() { - return Ok(RadonReport::from_result(Err(RadError::NoReveals), context)); + return Ok(RadonReport::from_result(Err(RadError::InsufficientQuorum), context)); } let items_to_tally = RadonTypes::from(RadonArray::from(radon_types_vec)); @@ -1529,7 +1529,7 @@ mod tests { ¤t_active_wips(), ); let report = res.unwrap().into_inner(); - let expected = RadonTypes::from(RadonError::try_from(RadError::NoReveals).unwrap()); + let expected = RadonTypes::from(RadonError::try_from(RadError::InsufficientQuorum).unwrap()); assert_eq!(report, expected); } diff --git a/validations/src/tests/tally_precondition.rs b/validations/src/tests/tally_precondition.rs index 142427662..b918216da 100644 --- a/validations/src/tests/tally_precondition.rs +++ b/validations/src/tests/tally_precondition.rs @@ -245,7 +245,7 @@ fn test_tally_precondition_clause_no_reveals() { let v = vec![]; let out = evaluate_tally_precondition_clause(v, 0.51, 1, ¤t_active_wips()).unwrap_err(); - assert_eq!(out, RadError::NoReveals); + assert_eq!(out, RadError::InsufficientQuorum); } #[test] diff --git a/wallet/src/repository/wallet/mod.rs b/wallet/src/repository/wallet/mod.rs index 6e401c586..16cd7ed98 100644 --- a/wallet/src/repository/wallet/mod.rs +++ b/wallet/src/repository/wallet/mod.rs @@ -2156,7 +2156,7 @@ fn build_tally_report( *pkh, model::Reveal { value: RadonTypes::from( - RadonError::try_from(RadError::NoReveals).unwrap(), + RadonError::try_from(RadError::InsufficientQuorum).unwrap(), ) .to_string(), in_consensus: false, From aec4c1a1f2fc9b648f3edaccba924b5cdcb2623b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 11:46:57 +0100 Subject: [PATCH 41/46] refactor(rad): *::InsufficientConsensus -> *::InsufficientMajority --- rad/src/conditions.rs | 12 ++++++------ validations/src/tests/tally_precondition.rs | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rad/src/conditions.rs b/rad/src/conditions.rs index eb5c93dd7..abd90539c 100644 --- a/rad/src/conditions.rs +++ b/rad/src/conditions.rs @@ -84,7 +84,7 @@ pub fn evaluate_tally_precondition_clause( } // If the achieved consensus is over the user-defined threshold, continue. - // Otherwise, return `RadError::InsufficientConsensus`. + // Otherwise, return `RadError::InsufficientMajority`. if achieved_consensus >= minimum_consensus { // Decide based on the most frequent type. match counter.max_pos { @@ -111,7 +111,7 @@ pub fn evaluate_tally_precondition_clause( let errors_array = RadonArray::from(errors); // Use the mode filter to get the count of the most common error. // That count must be greater than or equal to minimum_consensus, - // otherwise RadError::InsufficientConsensus is returned + // otherwise RadError::InsufficientMajority is returned let most_common_error_array = filters::mode::mode_filter(&errors_array, &mut ReportContext::default()); @@ -127,7 +127,7 @@ pub fn evaluate_tally_precondition_clause( _ => unreachable!("Mode of `RadonArray` containing only `RadonError`s cannot possibly be different from `RadonError`"), } } else { - Err(RadError::InsufficientConsensus { + Err(RadError::InsufficientMajority { achieved: achieved_consensus, required: minimum_consensus, }) @@ -139,7 +139,7 @@ pub fn evaluate_tally_precondition_clause( Err(RadError::ModeTie { values, max_count }) => { let achieved_consensus = f64::from(max_count) / num_commits_f; if achieved_consensus < minimum_consensus { - Err(RadError::InsufficientConsensus { + Err(RadError::InsufficientMajority { achieved: achieved_consensus, required: minimum_consensus, }) @@ -174,7 +174,7 @@ pub fn evaluate_tally_precondition_clause( } } } else { - Err(RadError::InsufficientConsensus { + Err(RadError::InsufficientMajority { achieved: achieved_consensus, required: minimum_consensus, }) @@ -214,7 +214,7 @@ pub fn evaluate_tally_postcondition_clause( // and not rewarded. let num_reveals = metadata.liars.len(); radon_report_from_error( - RadError::InsufficientConsensus { + RadError::InsufficientMajority { achieved: achieved_consensus, required: minimum_consensus, }, diff --git a/validations/src/tests/tally_precondition.rs b/validations/src/tests/tally_precondition.rs index b918216da..2b053e977 100644 --- a/validations/src/tests/tally_precondition.rs +++ b/validations/src/tests/tally_precondition.rs @@ -295,7 +295,7 @@ fn test_tally_precondition_clause_insufficient_consensus() { assert_eq!( out, - RadError::InsufficientConsensus { + RadError::InsufficientMajority { achieved: 0.5, required: 0.51, } @@ -304,7 +304,7 @@ fn test_tally_precondition_clause_insufficient_consensus() { #[test] fn test_tally_precondition_clause_errors_insufficient_consensus() { - // Two revealers that report two different errors result in `InsufficientConsensus` + // Two revealers that report two different errors result in `InsufficientMajority` // because there is only 50% consensus (1/2) let rad_err1 = RadonError::try_from(RadError::HttpStatus { status_code: 0 }).unwrap(); let rad_err2 = RadonError::try_from(RadError::RetrieveTimeout).unwrap(); @@ -322,7 +322,7 @@ fn test_tally_precondition_clause_errors_insufficient_consensus() { assert_eq!( out, - RadError::InsufficientConsensus { + RadError::InsufficientMajority { achieved: 0.5, required: 0.51, } From f7a190f0c0ad02d53d164ab18620b50efb682447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 11:48:59 +0100 Subject: [PATCH 42/46] refactor(rad): RadError::{*} -> RadError::Math{*} --- rad/src/operators/integer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rad/src/operators/integer.rs b/rad/src/operators/integer.rs index 5fda41039..12385b4f7 100644 --- a/rad/src/operators/integer.rs +++ b/rad/src/operators/integer.rs @@ -16,7 +16,7 @@ pub fn absolute(input: &RadonInteger) -> Result { if let Some(result) = result { Ok(RadonInteger::from(result)) } else { - Err(RadError::Overflow) + Err(RadError::MathOverflow) } } @@ -56,7 +56,7 @@ pub fn multiply(input: &RadonInteger, args: &[Value]) -> Result Result Ok(RadonInteger::from(x)), - None => Err(RadError::Overflow), + None => Err(RadError::MathOverflow), } } @@ -106,7 +106,7 @@ pub fn negate(input: &RadonInteger) -> Result { if let Some(result) = result { Ok(RadonInteger::from(result)) } else { - Err(RadError::Overflow) + Err(RadError::MathOverflow) } } @@ -124,7 +124,7 @@ pub fn power(input: &RadonInteger, args: &[Value]) -> Result Date: Wed, 24 Jan 2024 11:59:11 +0100 Subject: [PATCH 43/46] feat(rad): implement extended encoding rules for RadonTypes::RadonError --- rad/src/error.rs | 514 +++++++++++++++++++++++++++++++---------------- rad/src/lib.rs | 2 +- 2 files changed, 340 insertions(+), 176 deletions(-) diff --git a/rad/src/error.rs b/rad/src/error.rs index a0a64c341..b39a9ab69 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -6,6 +6,7 @@ use failure::{self, Fail}; use serde::{Serialize, Serializer}; use serde_cbor::value::Value as SerdeCborValue; +use witnet_data_structures::chain::tapi::ActiveWips; use witnet_data_structures::radon_error::{ErrorLike, RadonError, RadonErrors}; use crate::types::RadonTypes; @@ -445,6 +446,229 @@ pub enum RadError { impl RadError { + + pub fn into_error_code(&self) -> RadonErrors { + match self { + + RadError::CircumstantialFailure { .. } + | RadError::ArrayIndexOutOfBounds { .. } + | RadError::JsonPathNotFound { .. } + | RadError::HttpStatus { .. } + | RadError::MapKeyNotFound { .. } + | RadError::ModeTie { .. } + | RadError::RetrieveTimeout + | RadError::TallyExecution { .. } + => { + RadonErrors::CircumstantialFailure + } + RadError::InconsistentSource + | RadError::InconsistentSources + => { + RadonErrors::InconsistentSources + } + RadError::InsufficientCommits => RadonErrors::InsufficientCommits, + RadError::InsufficientMajority { .. } => RadonErrors::InsufficientMajority, + RadError::InsufficientQuorum => RadonErrors::InsufficientQuorum, + RadError::MalformedDataRequest { .. } + | RadError::ArrayFilterWrongSubscript { .. } + | RadError::BadSubscriptFormat { .. } + | RadError::HttpOther { .. } + | RadError::InvalidHttpBody { .. } + | RadError::InvalidHttpHeader { .. } + | RadError::InvalidScript { .. } + | RadError::JsonPathParse { .. } + | RadError::NoOperatorInCompoundCall + | RadError::NotIntegerOperator + | RadError::NotNaturalOperator { .. } + | RadError::RequestTooManySources + | RadError::ScriptNotArray { .. } + | RadError::ScriptTooManyCalls { .. } + | RadError::SourceScriptNotArray + | RadError::SourceScriptNotCBOR + | RadError::SourceScriptNotRADON + | RadError::UnknownFilter { .. } + | RadError::UnknownOperator { .. } + | RadError::UnknownReducer { .. } + | RadError::UnknownRetrieval { .. } + | RadError::UnsupportedEncodingSchema { .. } + | RadError::UnsupportedFilter { .. } + | RadError::UnsupportedFilterInAT { .. } + | RadError::UnsupportedHashFunction { .. } + | RadError::UnsupportedOperator { .. } + | RadError::UnsupportedOperatorInTally { .. } + | RadError::UnsupportedOpNonHomogeneous { .. } + | RadError::UnsupportedReducer { .. } + | RadError::UnsupportedReducerInAT { .. } + | RadError::UrlParseError { .. } + | RadError::WrongArguments { .. } + => { + RadonErrors::MalformedDataRequest + } + RadError::MalformedResponses { .. } + | RadError::BufferIsNotValue { .. } + | RadError::Decode { .. } + | RadError::DifferentSizeArrays { .. } + | RadError::EmptyArray + | RadError::Encode { .. } + | RadError::EncodeReveal + | RadError::Hash + | RadError::InvalidHttpResponse { .. } + | RadError::JsonParse { .. } + | RadError::MalformedReveal + | RadError::MathDivisionByZero + | RadError::MathOverflow + | RadError::MathUnderflow + | RadError::MismatchingTypes { .. } + | RadError::ParseBool { .. } + | RadError::ParseFloat { .. } + | RadError::ParseInt { .. } + | RadError::UnsupportedSortOp { .. } + | RadError::XmlParse { .. } + | RadError::XmlParseOverflow + => { + RadonErrors::MalformedResponses + } + RadError::OversizedTallyResult { .. } => { + RadonErrors::OversizedTallyResult + } + RadError::Subscript { inner, .. } => { + inner.into_error_code() + } + RadError::DecodeRadonErrorArgumentsRadonTypesFail { .. } + | RadError::DecodeRadonErrorBadCode { .. } + | RadError::DecodeRadonErrorEmptyArray + | RadError::DecodeRadonErrorMissingArguments + | RadError::DecodeRadonErrorNotArray { .. } + | RadError::DecodeRadonErrorUnknownCode { .. } + | RadError::DecodeRadonErrorWrongArguments { .. } + | RadError::EncodeRadonErrorArguments { .. } + | RadError::EncodeRadonErrorUnknownCode + | RadError::UnhandledIntercept { .. } + | RadError::UnhandledInterceptV2 { .. } + | RadError::Unknown + => { + RadonErrors::UnhandledIntercept + } + } + } + + pub fn into_error_subcode(&self) -> RadonErrors { + match self { + + RadError::ArrayIndexOutOfBounds { .. } => RadonErrors::ArrayIndexOutOfBounds, + RadError::ArrayFilterWrongSubscript { .. } + | RadError::UnsupportedSortOp { .. } + => { + RadonErrors::WrongSubscriptInput + } + RadError::BadSubscriptFormat { .. } + | RadError::InvalidScript { .. } + | RadError::ScriptNotArray { .. } + | RadError::SourceScriptNotArray + => { + RadonErrors::SourceScriptNotArray + } + RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue, + RadError::CircumstantialFailure { subcode, .. } => { + RadonErrors::try_from(*subcode).unwrap_or_default() + } + RadError::Decode { .. } => RadonErrors::Decode, + RadError::DecodeRadonErrorArgumentsRadonTypesFail { .. } + | RadError::DecodeRadonErrorBadCode { .. } + | RadError::DecodeRadonErrorEmptyArray { .. } + | RadError::DecodeRadonErrorMissingArguments { .. } + | RadError::DecodeRadonErrorNotArray { .. } + | RadError::DecodeRadonErrorUnknownCode { .. } + | RadError::DecodeRadonErrorWrongArguments { .. } + | RadError::MalformedReveal + => { + RadonErrors::MalformedReveals + } + RadError::DifferentSizeArrays { .. } => RadonErrors::MismatchingArrays, + RadError::EmptyArray => RadonErrors::EmptyArray, + RadError::Encode { .. } => RadonErrors::Encode, + RadError::EncodeReveal + | RadError::EncodeRadonErrorArguments { .. } + | RadError::EncodeRadonErrorUnknownCode { .. } + => { + RadonErrors::EncodeReveals + } + // RadError::Filter { .. } => RadonErrors::Filter, + RadError::Hash => RadonErrors::Hash, + RadError::HttpStatus { .. } + | RadError::HttpOther { .. } + => { + RadonErrors::HttpErrors + } + RadError::InvalidHttpBody { .. } => RadonErrors::SourceRequestBody, + RadError::InvalidHttpHeader { .. } => RadonErrors::SourceRequestHeaders, + RadError::JsonParse { .. } + | RadError::JsonPathParse { .. } + | RadError::XmlParse { .. } + | RadError::ParseBool { .. } + | RadError::ParseFloat { .. } + | RadError::ParseInt { .. } + => { + RadonErrors::Parse + } + RadError::JsonPathNotFound { .. } => RadonErrors::JsonPathNotFound, + RadError::MalformedDataRequest { subcode } => { + RadonErrors::try_from(*subcode).unwrap_or_default() + } + RadError::MalformedResponses { subcode } => { + RadonErrors::try_from(*subcode).unwrap_or_default() + } + RadError::MapKeyNotFound { .. } => RadonErrors::MapKeyNotFound, + RadError::MathDivisionByZero => RadonErrors::MathDivisionByZero, + RadError::MathOverflow => RadonErrors::MathOverflow, + RadError::MathUnderflow => RadonErrors::MathUnderflow, + RadError::MismatchingTypes { .. } + | RadError::WrongArguments { .. } + => { + RadonErrors::WrongArguments + } + RadError::ModeTie { .. } => RadonErrors::ModeTie, + // RadError::Reduce { .. } => RadonErrors::Reduce, + RadError::RequestTooManySources => RadonErrors::RequestTooManySources, + RadError::RetrieveTimeout => RadonErrors::RetrievalsTimeout, + RadError::ScriptTooManyCalls { .. } => RadonErrors::ScriptTooManyCalls, + RadError::SourceScriptNotCBOR { .. } => RadonErrors::SourceScriptNotCBOR, + RadError::SourceScriptNotRADON { .. } => RadonErrors::SourceScriptNotRADON, + RadError::Subscript { inner, .. } => { + inner.as_ref().into_error_subcode() + } + RadError::UnknownFilter { .. } + | RadError::UnsupportedFilter { .. } + | RadError::UnsupportedFilterInAT { .. } + => { + RadonErrors::UnsupportedFilter + } + RadError::UnknownOperator { .. } + | RadError::UnsupportedOperator { .. } + | RadError::UnsupportedOperatorInTally { .. } + | RadError::NoOperatorInCompoundCall { .. } + | RadError::NotIntegerOperator + | RadError::NotNaturalOperator { .. } + => { + RadonErrors::UnsupportedOperator + } + RadError::UnknownReducer { .. } + | RadError::UnsupportedReducer { .. } + | RadError::UnsupportedReducerInAT { .. } + => { + RadonErrors::UnsupportedReducer + } + RadError::UnknownRetrieval => RadonErrors::UnsupportedRequestType, + RadError::UnsupportedHashFunction { .. } => RadonErrors::UnsupportedHashFunction, + RadError::UnsupportedOpNonHomogeneous { .. } => RadonErrors::NonHomogeneousArrays, + RadError::UrlParseError { .. } => RadonErrors::SourceRequestURL, + RadError::XmlParseOverflow => RadonErrors::ParseOverflow, + _ => { + RadonErrors::Unknown + } + } + } + pub fn try_from_cbor_array( serde_cbor_array: Vec, ) -> Result, RadError> { @@ -465,7 +689,7 @@ impl RadError { Some(tail.to_vec()) }; - Ok(RadError::try_from_kind_and_cbor_args( + Ok(RadError::try_into_radon_error( kind, serde_cbor_error_args, )?) @@ -476,161 +700,76 @@ impl RadError { } } None => { - println!("None"); Err(RadError::DecodeRadonErrorEmptyArray) } } } - pub fn try_from_kind_and_cbor_args( - kind: RadonErrors, - error_args: Option>, - ) -> Result, RadError> { - // TODO: we currently allow extra arguments when the RadError does not expect any arguments - fn deserialize_args( - error_args: Option>, - ) -> Result { - let error_args = if let Some(x) = error_args { - SerdeCborValue::Array(x) - } else { - return Err(RadError::DecodeRadonErrorMissingArguments); - }; - - serde_cbor::value::from_value(error_args.clone()).map_err(|e| { - RadError::DecodeRadonErrorWrongArguments { - arguments: Some(error_args), - message: e.to_string(), + pub fn try_into_cbor_array(&self) -> Result, RadError> { + let (kind, subcode) = (self.into_error_code(), u8::from(self.into_error_subcode())); + let args = match kind { + + RadonErrors::CircumstantialFailure => { + match self { + RadError::ArrayIndexOutOfBounds { index } => Some(legacy::serialize_args((subcode, index,))?), + RadError::CircumstantialFailure { subcode, args } => Some(legacy::serialize_args((subcode, args))?), + RadError::JsonPathNotFound { path } => Some(legacy::serialize_args((subcode, path,))?), + RadError::HttpStatus { status_code } => Some(legacy::serialize_args((subcode, status_code,))?), + RadError::MapKeyNotFound { key } => Some(legacy::serialize_args((subcode, key,))?), + RadError::TallyExecution { inner, message } => { + let message = match (inner, message) { + // Only serialize the message + (_, Some(message)) => message.clone(), + // But if there is no message, serialize the debug representation of inner + (Some(inner), None) => format!("inner: {:?}", inner), + // And if there is no inner, serialize this string + (None, None) => "inner: None".to_string(), + }; + Some(legacy::serialize_args((subcode, message,))?) + } + _ => Some(legacy::serialize_args((u8::from(subcode),))?) } - }) - } - - Ok(RadonError::new(match kind { - RadonErrors::BufferIsNotValue => { - let (description,) = deserialize_args(error_args)?; - RadError::BufferIsNotValue { description } - } - RadonErrors::RequestTooManySources => RadError::RequestTooManySources, - RadonErrors::ScriptTooManyCalls => RadError::ScriptTooManyCalls, - RadonErrors::Overflow => RadError::Overflow, - RadonErrors::InsufficientCommits => RadError::InsufficientCommits, - RadonErrors::NoReveals => RadError::NoReveals, - RadonErrors::SourceScriptNotCBOR => RadError::SourceScriptNotCBOR, - RadonErrors::SourceScriptNotArray => RadError::SourceScriptNotArray, - RadonErrors::SourceScriptNotRADON => RadError::SourceScriptNotRADON, - RadonErrors::Underflow => RadError::Underflow, - RadonErrors::DivisionByZero => RadError::DivisionByZero, - RadonErrors::RetrieveTimeout => RadError::RetrieveTimeout, - RadonErrors::MalformedReveal => RadError::MalformedReveal, - RadonErrors::EncodeReveal => RadError::EncodeReveal, - RadonErrors::ArrayIndexOutOfBounds => { - let (index,) = deserialize_args(error_args)?; - RadError::ArrayIndexOutOfBounds { index } } - RadonErrors::MapKeyNotFound => { - let (key,) = deserialize_args(error_args)?; - RadError::MapKeyNotFound { key } + + RadonErrors::InconsistentSources | RadonErrors::InsufficientCommits | RadonErrors::InsufficientQuorum => { + None } - RadonErrors::UnsupportedOperator => { - let (input_type, operator, args) = deserialize_args(error_args)?; - RadError::UnsupportedOperator { - input_type, - operator, - args, + + RadonErrors::InsufficientMajority => { + if let RadError::InsufficientMajority { achieved, required } = self { + Some(legacy::serialize_args((achieved, required,))?) + } else { + None } } - RadonErrors::HTTPError => { - let (status_code,) = deserialize_args(error_args)?; - RadError::HttpStatus { status_code } - } - RadonErrors::InsufficientConsensus => { - let (achieved, required) = deserialize_args(error_args)?; - RadError::InsufficientConsensus { achieved, required } - } - RadonErrors::TallyExecution => { - let (message,) = deserialize_args(error_args)?; - RadError::TallyExecution { - inner: None, - message: Some(message), - } + + RadonErrors::MalformedDataRequest | RadonErrors::MalformedResponses => { + Some(legacy::serialize_args((subcode, ))?) } - RadonErrors::UnhandledIntercept => { - if error_args.is_none() { - RadError::UnhandledInterceptV2 { inner: None } + + RadonErrors::OversizedTallyResult => { + if let RadError::OversizedTallyResult { actual, allowance } = self { + Some(legacy::serialize_args((actual, allowance,))?) } else { - let (message,) = deserialize_args(error_args)?; - RadError::UnhandledIntercept { - inner: None, - message: Some(message), - } + None } } - RadonErrors::Unknown => RadError::Unknown, - // The only case where a Bridge RadonError could be included in the protocol is that - // if a witness node report as a reveal, and in that case it would be considered - // as a MalformedReveal - RadonErrors::BridgeMalformedRequest - | RadonErrors::BridgePoorIncentives - | RadonErrors::BridgeOversizedResult => RadError::MalformedReveal, - })) - } - - pub fn try_into_cbor_array(&self) -> Result, RadError> { - fn serialize_args( - args: T, - ) -> Result { - serde_cbor::value::to_value(&args).map_err(|_| RadError::EncodeRadonErrorArguments { - error_args: format!("{:?}", args), - }) - } - let kind = u8::from(self.try_into_error_code()?); - - let args = match self { - RadError::UnsupportedOperator { - input_type, - operator, - args, - } => Some(serialize_args((input_type, operator, args))?), - RadError::HttpStatus { status_code } => Some(serialize_args((status_code,))?), - RadError::InsufficientConsensus { achieved, required } => { - Some(serialize_args((achieved, required))?) - } - RadError::TallyExecution { inner, message } => { - let message = match (inner, message) { - // Only serialize the message - (_, Some(message)) => message.clone(), - // But if there is no message, serialize the debug representation of inner - (Some(inner), None) => format!("inner: {:?}", inner), - // And if there is no inner, serialize this string - (None, None) => "inner: None".to_string(), - }; - Some(serialize_args((message,))?) - } - RadError::ArrayIndexOutOfBounds { index } => Some(serialize_args((index,))?), - RadError::MapKeyNotFound { key } => Some(serialize_args((key,))?), - RadError::UnhandledIntercept { inner, message } => { - let message = match (inner, message) { - // Only serialize the message - (_, Some(message)) => message.clone(), - // But if there is no message, serialize the debug representation of inner - (Some(inner), None) => { - // Fix #1993 by emulating a bug from old versions of Rust (rust-lang/rust#83046) - if_rust_version::if_rust_version! { >= 1.53 { - format!("inner: {:?}", inner).replace('\'', "\\'") - } else { - format!("inner: {:?}", inner) - }} - } - // And if there is no inner, serialize this string - (None, None) => "inner: None".to_string(), - }; - Some(serialize_args((message,))?) + _ => { + // Unhandled intercepts must be serialized without a subcode + let message = if_rust_version::if_rust_version! { >= 1.53 { + format!("{:?}", self).replace('\'', "\\'") + } else { + format!("{:?}", self) + }}; + Some(legacy::serialize_args((message,))?) } - RadError::BufferIsNotValue { description } => Some(serialize_args((description,))?), - _ => None, }; - let mut v = vec![SerdeCborValue::Integer(i128::from(kind))]; + let mut v = vec![ + SerdeCborValue::Integer(i128::from(u8::from(kind))), + SerdeCborValue::Integer(i128::from(subcode)), + ]; match args { None => {} @@ -650,44 +789,69 @@ impl RadError { Ok(v) } - pub fn try_into_error_code(&self) -> Result { - Ok(match self { - RadError::Unknown => RadonErrors::Unknown, - RadError::BufferIsNotValue { .. } => RadonErrors::BufferIsNotValue, - RadError::SourceScriptNotCBOR => RadonErrors::SourceScriptNotCBOR, - RadError::SourceScriptNotArray => RadonErrors::SourceScriptNotArray, - RadError::SourceScriptNotRADON => RadonErrors::SourceScriptNotRADON, - RadError::RequestTooManySources => RadonErrors::RequestTooManySources, - RadError::ScriptTooManyCalls => RadonErrors::ScriptTooManyCalls, - RadError::UnsupportedOperator { .. } => RadonErrors::UnsupportedOperator, - RadError::HttpStatus { .. } => RadonErrors::HTTPError, - RadError::Underflow => RadonErrors::Underflow, - RadError::Overflow => RadonErrors::Overflow, - RadError::DivisionByZero => RadonErrors::DivisionByZero, - RadError::InsufficientCommits => RadonErrors::InsufficientCommits, - RadError::NoReveals => RadonErrors::NoReveals, - RadError::RetrieveTimeout => RadonErrors::RetrieveTimeout, - RadError::InsufficientConsensus { .. } => RadonErrors::InsufficientConsensus, - RadError::TallyExecution { .. } => RadonErrors::TallyExecution, - RadError::UnhandledIntercept { .. } | RadError::UnhandledInterceptV2 { .. } => { - RadonErrors::UnhandledIntercept + pub fn try_into_radon_error( + kind: RadonErrors, + error_args: Option> + ) -> Result, Self> { + Ok(RadonError::new(match kind { + + RadonErrors::CircumstantialFailure => { + match error_args.unwrap_or_default().split_first() { + Some((head, tail)) => { + if let SerdeCborValue::Integer(error_code) = head { + let error_code = u8::try_from(*error_code).map_err(|_| { + RadError::DecodeRadonErrorBadCode { + actual_type: format!("{:?}", error_code), + } + })?; + + let subcode = RadonErrors::try_from(error_code) + .map_err(|_| RadError::DecodeRadonErrorUnknownCode { error_code })?; + + let subcode_args = if tail.is_empty() { + None + } else { + Some(tail.to_vec()) + }; + + RadError::CircumstantialFailure { + subcode: u8::from(subcode), + args: subcode_args + } + } else { + return Err(RadError::DecodeRadonErrorBadCode { + actual_type: format!("{:?}", head), + }); + } + } + None => { + return Err(RadError::DecodeRadonErrorEmptyArray); + } + } } - RadError::MalformedReveal => RadonErrors::MalformedReveal, - RadError::EncodeReveal => RadonErrors::EncodeReveal, - RadError::ArrayIndexOutOfBounds { .. } => RadonErrors::ArrayIndexOutOfBounds, - RadError::MapKeyNotFound { .. } => RadonErrors::MapKeyNotFound, - // The `InconsistentSource` error is mapped here for the sake of backwards - // compatibility. Namely, to enable paranoid retrieval without having to immediately - // introduce a breaking change that may jeopardize oracle queries. The point of making - // the mapping here is to only affect actual witnessing and committing, but not the - // `try_data_request` function, which can rather use the `InconsistentSource` error for - // clarity. - // TODO: pursue a WIP that introduces `InconsistentSource` as a proper - // RadonError at the protocol level - // https://github.com/witnet/WIPs/issues/86 - RadError::InconsistentSource => RadonErrors::Unknown, - _ => return Err(RadError::EncodeRadonErrorUnknownCode), - }) + + RadonErrors::InconsistentSources => RadError::InconsistentSource, + + RadonErrors::MalformedDataRequest => { + let (subcode, ) = legacy::deserialize_args(error_args)?; + RadError::MalformedDataRequest { subcode } + } + + RadonErrors::MalformedResponses => { + let (subcode, ) = legacy::deserialize_args(error_args)?; + RadError::MalformedResponses { subcode } + } + + RadonErrors::OversizedTallyResult => { + let (actual, allowance,) = legacy::deserialize_args(error_args)?; + RadError::OversizedTallyResult { actual, allowance} + } + + _ => { + return Self::try_into_radon_error_before_wip0028(kind, error_args); + } + + })) } /// Replaces the `from` field in instances of `RadError::Decode` diff --git a/rad/src/lib.rs b/rad/src/lib.rs index b0ad3dc2b..0688ddb4a 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -1543,7 +1543,7 @@ mod tests { let int = RadonTypes::from(RadonInteger::from(0)); // RadonError with error code 0 let kind = RadonErrors::try_from(0).unwrap(); - let err = RadonTypes::from(RadError::try_from_kind_and_cbor_args(kind, None).unwrap()); + let err = RadonTypes::from(RadError::try_into_radon_error(kind, None).unwrap()); // Ensure they encoded differently (errors are tagged using `39` as CBOR tag) assert_ne!(int.encode_legacy(), err.encode_legacy()); // And they are not equal in runtime either From 521d7a0b3c906c7174846eb636f13ce0da3e6d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 12:10:30 +0100 Subject: [PATCH 44/46] chore(rad): revert just-not-yet error cases --- data_structures/src/radon_error.rs | 9 +----- rad/src/error.rs | 47 ------------------------------ 2 files changed, 1 insertion(+), 55 deletions(-) diff --git a/data_structures/src/radon_error.rs b/data_structures/src/radon_error.rs index ab9771f9b..9cd699c63 100644 --- a/data_structures/src/radon_error.rs +++ b/data_structures/src/radon_error.rs @@ -51,8 +51,6 @@ pub enum RadonErrors { UnsupportedReducer = 0x23, /// Some Radon request type is not currently supported. UnsupportedRequestType = 0x24, - /// Some Radon encoding function is not currently supported. - UnsupportedEncodingFunction = 0x25, /// Wrong number (or type) of arguments were passed to some Radon operator. WrongArguments = 0x28, @@ -113,10 +111,7 @@ pub enum RadonErrors { /// Values returned from a majority of data sources did not match the expected schema: MalformedResponses = 0x56, /// The data request was not properly formated: - MalformedDataRequest = 0x57, - /// The size of serialized tally result exceeds allowance: - OversizedTallyResult = 0x5F, - + MalformedDataRequest = 0x57, /////////////////////////////////////////////////////////////////////////// // Inter-stage runtime error sub-codes @@ -133,8 +128,6 @@ pub enum RadonErrors { ArrayIndexOutOfBounds = 0x70, /// Tried to access a value from a map using a key that does not exist. MapKeyNotFound = 0x71, - /// Tried to extract value from a map using a JSON Path that returns no values. - JsonPathNotFound = 0x72, /////////////////////////////////////////////////////////////////////////// // Inter-client first-order error codes. diff --git a/rad/src/error.rs b/rad/src/error.rs index b39a9ab69..6842b0d1a 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -167,15 +167,6 @@ pub enum RadError { description )] JsonParse { description: String }, - /// Failed to parse a JSON path selector from a string value - #[fail( - display = "Failed to parse a JSON path from a string value: {:?}", - description - )] - JsonPathParse { description: String }, - /// The given JSON path is not present in a JSON-stringified object - #[fail(display = "Failed to find JSON path `{}` from RadonString", path)] - JsonPathNotFound { path: String }, /// Invalid reveal serialization (malformed reveals are converted to this value) #[fail(display = "The reveal was not serialized correctly")] MalformedReveal, @@ -283,12 +274,6 @@ pub enum RadError { /// The given retrieval code is unknown #[fail(display = "Retrieval code is unknown")] UnknownRetrieval, - /// The given encoding schema is not implemented - #[fail( - display = "Encoding schema #{} not implemented", - schema - )] - UnsupportedEncodingSchema { schema: usize }, /// The given filter is not implemented for the type of the input Array #[fail( display = "Filter `{}` is not implemented for Array `{:?}`", @@ -408,13 +393,6 @@ pub enum RadError { subcode )] MalformedResponses { subcode: u8 }, - - /// Size of serialized tally result exceeds allowance: - #[fail( - display = "Tally result size exceeded allowance (actual: {}, allowance: {})", - actual, allowance - )] - OversizedTallyResult { actual: usize, allowance: usize }, /// `RadError` cannot be converted to `RadonError` but it should, because it is needed for the tally result #[fail( @@ -452,7 +430,6 @@ impl RadError { RadError::CircumstantialFailure { .. } | RadError::ArrayIndexOutOfBounds { .. } - | RadError::JsonPathNotFound { .. } | RadError::HttpStatus { .. } | RadError::MapKeyNotFound { .. } | RadError::ModeTie { .. } @@ -476,7 +453,6 @@ impl RadError { | RadError::InvalidHttpBody { .. } | RadError::InvalidHttpHeader { .. } | RadError::InvalidScript { .. } - | RadError::JsonPathParse { .. } | RadError::NoOperatorInCompoundCall | RadError::NotIntegerOperator | RadError::NotNaturalOperator { .. } @@ -490,7 +466,6 @@ impl RadError { | RadError::UnknownOperator { .. } | RadError::UnknownReducer { .. } | RadError::UnknownRetrieval { .. } - | RadError::UnsupportedEncodingSchema { .. } | RadError::UnsupportedFilter { .. } | RadError::UnsupportedFilterInAT { .. } | RadError::UnsupportedHashFunction { .. } @@ -528,9 +503,6 @@ impl RadError { => { RadonErrors::MalformedResponses } - RadError::OversizedTallyResult { .. } => { - RadonErrors::OversizedTallyResult - } RadError::Subscript { inner, .. } => { inner.into_error_code() } @@ -603,7 +575,6 @@ impl RadError { RadError::InvalidHttpBody { .. } => RadonErrors::SourceRequestBody, RadError::InvalidHttpHeader { .. } => RadonErrors::SourceRequestHeaders, RadError::JsonParse { .. } - | RadError::JsonPathParse { .. } | RadError::XmlParse { .. } | RadError::ParseBool { .. } | RadError::ParseFloat { .. } @@ -611,7 +582,6 @@ impl RadError { => { RadonErrors::Parse } - RadError::JsonPathNotFound { .. } => RadonErrors::JsonPathNotFound, RadError::MalformedDataRequest { subcode } => { RadonErrors::try_from(*subcode).unwrap_or_default() } @@ -713,7 +683,6 @@ impl RadError { match self { RadError::ArrayIndexOutOfBounds { index } => Some(legacy::serialize_args((subcode, index,))?), RadError::CircumstantialFailure { subcode, args } => Some(legacy::serialize_args((subcode, args))?), - RadError::JsonPathNotFound { path } => Some(legacy::serialize_args((subcode, path,))?), RadError::HttpStatus { status_code } => Some(legacy::serialize_args((subcode, status_code,))?), RadError::MapKeyNotFound { key } => Some(legacy::serialize_args((subcode, key,))?), RadError::TallyExecution { inner, message } => { @@ -747,14 +716,6 @@ impl RadError { Some(legacy::serialize_args((subcode, ))?) } - RadonErrors::OversizedTallyResult => { - if let RadError::OversizedTallyResult { actual, allowance } = self { - Some(legacy::serialize_args((actual, allowance,))?) - } else { - None - } - } - _ => { // Unhandled intercepts must be serialized without a subcode let message = if_rust_version::if_rust_version! { >= 1.53 { @@ -841,11 +802,6 @@ impl RadError { let (subcode, ) = legacy::deserialize_args(error_args)?; RadError::MalformedResponses { subcode } } - - RadonErrors::OversizedTallyResult => { - let (actual, allowance,) = legacy::deserialize_args(error_args)?; - RadError::OversizedTallyResult { actual, allowance} - } _ => { return Self::try_into_radon_error_before_wip0028(kind, error_args); @@ -1216,9 +1172,6 @@ mod tests { RadonErrors::MapKeyNotFound => RadError::MapKeyNotFound { key: String::from("value"), }, - RadonErrors::JsonPathNotFound => RadError::JsonPathNotFound { - path: String::from("*.*") - }, RadonErrors::UnhandledIntercept => RadError::UnhandledIntercept { inner: None, message: Some("Only the message field is serialized".to_string()), From 13225c81c9ffb55d7e63b82edb16aeb596eca99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 12:43:54 +0100 Subject: [PATCH 45/46] fix(rad): RadError::HttpOther must be mapped to RadonErrors::CircumstantialFailure --- rad/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rad/src/error.rs b/rad/src/error.rs index 6842b0d1a..0cc6cc0c8 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -430,6 +430,7 @@ impl RadError { RadError::CircumstantialFailure { .. } | RadError::ArrayIndexOutOfBounds { .. } + | RadError::HttpOther { .. } | RadError::HttpStatus { .. } | RadError::MapKeyNotFound { .. } | RadError::ModeTie { .. } @@ -449,7 +450,6 @@ impl RadError { RadError::MalformedDataRequest { .. } | RadError::ArrayFilterWrongSubscript { .. } | RadError::BadSubscriptFormat { .. } - | RadError::HttpOther { .. } | RadError::InvalidHttpBody { .. } | RadError::InvalidHttpHeader { .. } | RadError::InvalidScript { .. } From a8d0b577b49a691999b70ffb281c595e72ea2934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20D=C3=ADaz?= Date: Wed, 24 Jan 2024 12:46:52 +0100 Subject: [PATCH 46/46] feat(rad): implement RadError::{InvalidHttpBody, InvalidHttpResponse} --- rad/src/lib.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 0688ddb4a..004e9e309 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -266,13 +266,12 @@ async fn http_response( for (name, value) in &retrieve.headers { // Handle invalid header names and values with a specific and friendly error message validate_header(name, value)?; - builder = builder.header(name, value); } // Finally attach the body to complete building the HTTP request - builder.body(body).map_err(|e| RadError::HttpOther { - message: e.to_string(), + builder.body(body).map_err(|e| RadError::InvalidHttpBody { + error: e.to_string(), }) })?; @@ -280,8 +279,8 @@ async fn http_response( let response = client .send(request) .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), + .map_err(|err| RadError::InvalidHttpResponse { + error: err.to_string() })? .inner(); @@ -301,8 +300,8 @@ async fn http_response( // todo: before reading the response buffer, an error should be thrown if it was too big body.read_to_end(&mut response_bytes) .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), + .map_err(|x| RadError::InvalidHttpResponse { + error: x.to_string(), })?; response = RadonTypes::from(RadonBytes::from(response_bytes)); } else { @@ -315,8 +314,8 @@ async fn http_response( _ => { body.read_to_string(&mut response_string) .await - .map_err(|x| RadError::HttpOther { - message: x.to_string(), + .map_err(|x| RadError::InvalidHttpResponse { + error: x.to_string(), })?; } }