feat: populate criteria (#42)

populate criteria missing from the `catalogue.json`

Cargo.toml

@@ -24,6 +24,9 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 reqwest = { version = "0.12", default-features = false, features = ["stream", "default-tls"] }
 tower-http = { version = "0.6", features = ["cors"] }
 
+[dev-dependencies]
+pretty_assertions = "1.4.0"
+
 [build-dependencies]
 build-data = "0.2.1"
 

src/catalogue.rs

@@ -1,3 +1,5 @@
+use serde::Deserialize;
+use serde::Serialize;
 use std::{collections::BTreeMap, sync::Arc, time::Duration};
 
 use reqwest::Url;
@@ -11,36 +13,64 @@ pub type CriteriaGroup = BTreeMap<String, Criteria>;
 
 pub type CriteriaGroups = BTreeMap<String, CriteriaGroup>;
 
-fn get_element<'a>(count: &'a CriteriaGroups, key1: &'a str, key2: &'a str, key3: &'a str) -> Option<&'a u64> {
-    count.get(key1)
-        .and_then(|group| group.get(key2))
-        .and_then(|criteria| criteria.get(key3))
+#[derive(Serialize, Deserialize, Debug, Clone)]
+struct CatalogueCriterion {
+    key: String,
+    name: String,
+    description: String,
+    count: u64,
+}
+
+fn get_element<'a>(
+    counts: &'a CriteriaGroups,
+    key1: &'a str,
+    key2: &'a str,
+    key3: &'a str,
+) -> Option<&'a u64> {
+    counts
+        .get(key1)?
+        .get(key2)?
+        .get(key3)
+}
+
+fn get_stratifier<'a>(
+    counts: &'a CriteriaGroups,
+    key1: &'a str,
+    key2: &'a str,
+) -> Option<&'a Criteria> {
+    counts
+    .get(key1)?
+    .get(key2)
+
 }
 
 pub fn spawn_thing(catalogue_url: Url, prism_url: Url) -> Arc<Mutex<Value>> {
     let thing: Arc<Mutex<Value>> = Arc::default();
     let thing1 = thing.clone();
     tokio::spawn(async move {
-        tokio::time::sleep(Duration::from_secs(10)).await;    
+        tokio::time::sleep(Duration::from_secs(10)).await;
         loop {
             match get_extended_json(catalogue_url.clone(), prism_url.clone()).await {
                 Ok(new_value) => {
                     *thing1.lock().await = new_value;
                     info!("Updated Catalogue!");
                     tokio::time::sleep(Duration::from_secs(60 * 60)).await;
-                },
+                }
                 Err(err) => {
                     warn!("Failed to get thing: {err}.\n Retrying in 5s.");
                     tokio::time::sleep(Duration::from_secs(5)).await;
-                },
+                }
             }
         }
     });
 
     thing
 }
 
-pub async fn get_extended_json(catalogue_url: Url, prism_url: Url) -> Result<Value, reqwest::Error> {
+pub async fn get_extended_json(
+    catalogue_url: Url,
+    prism_url: Url,
+) -> Result<Value, reqwest::Error> {
     debug!("Fetching catalogue from {catalogue_url} ...");
 
     let resp = reqwest::Client::new()
@@ -51,7 +81,6 @@ pub async fn get_extended_json(catalogue_url: Url, prism_url: Url) -> Result<Val
 
     let mut json: Value = resp.json().await?;
 
-
     let prism_resp = reqwest::Client::new()
         .post(format!("{}criteria", prism_url))
         .header("Content-Type", "application/json")
@@ -78,22 +107,23 @@ fn recurse(json: &mut Value, counts: &mut CriteriaGroups) {
             for ele in arr {
                 recurse(ele, counts);
             }
-        },
+        }
         Value::Object(obj) => {
-
-            if ! obj.contains_key("childCategories") {
+            if !obj.contains_key("childCategories") {
                 for (_key, child_val) in obj.iter_mut() {
                     recurse(child_val, counts);
                 }
             } else {
                 let group_key = obj.get("key").expect("Got JSON element with childCategories but without (group) key. Please check json.").as_str()
                 .expect("Got JSON where a criterion key was not a string. Please check json.").to_owned();
 
-                //TODO consolidate catalogue and MeasureReport group names
-                let group_key = if group_key == "patient" {"patients"} 
-                else if group_key == "tumor_classification" {"diagnosis"} 
-                else if group_key == "biosamples" {"specimen"}
-                else {&group_key};
+                //TODO consolidate catalogue and MeasureReport group names, also between projects
+                let group_key = match group_key.as_str() {
+                    "patient" | "donor" => "patients",
+                    "tumor_classification" => "diagnosis",
+                    "biosamples" | "sample" => "specimen",
+                    _ => &group_key
+                };
 
                 let children_cats = obj
                     .get_mut("childCategories")
@@ -102,39 +132,100 @@ fn recurse(json: &mut Value, counts: &mut CriteriaGroups) {
                     .unwrap()
                     .iter_mut()
                     .filter(|item| item.get("type").unwrap_or(&Value::Null) == "EQUALS");
-                
+
                 for child_cat in children_cats {
                     let stratifier_key = child_cat.get("key").expect("Got JSON element with childCategory that does not contain a (stratifier) key. Please check json.").as_str()
                     .expect("Got JSON where a criterion key was not a string. Please check json.").to_owned();
 
+                    //TODO consolidate catalogue and MeasureReport group names, also between projects
+                    let stratifier_key = if stratifier_key == "gender" {
+                        "Gender"
+                    } else {
+                        &stratifier_key
+                    };
+
                     let criteria = child_cat
                         .get_mut("criteria")
                         .expect("Got JSON element with childCategory that does not contain a criteria array. Please check json.")
                         .as_array_mut()
                         .expect("Got JSON element with childCategory with criteria that are not an array. Please check json.");
 
-                    for criterion in criteria {
-                        let criterion = criterion.as_object_mut()
-                            .expect("Got JSON where a criterion was not an object. Please check json.");
-                        let stratum_key = criterion.get("key")
+                    if !criteria.is_empty() {
+                        for criterion in criteria {
+                            let criterion = criterion.as_object_mut().expect(
+                                "Got JSON where a criterion was not an object. Please check json.",
+                            );
+                            let stratum_key = criterion.get("key")
                             .expect("Got JSON where a criterion did not have a key. Please check json.")
                             .as_str()
                             .expect("Got JSON where a criterion key was not a string. Please check json.");
-
-                        let count_from_prism = get_element(counts, &group_key, &stratifier_key, stratum_key);
-
-                        match count_from_prism {
-                            Some(count) => {
-                                criterion.insert("count".into(), json!(count));
-                            },
-                            None => {
-                                debug!("No count from Prism for {}, {}, {}", group_key, stratifier_key, stratum_key);
+
+                            let count_from_prism =
+                                get_element(counts, &group_key, &stratifier_key, stratum_key);
+
+                            match count_from_prism {
+                                Some(count) => {
+                                    criterion.insert("count".into(), json!(count));
+                                }
+                                None => {
+                                    debug!(
+                                        "No count from Prism for {}, {}, {}",
+                                        group_key, stratifier_key, stratum_key
+                                    );
+                                }
+                            }
+                        }
+                    } else {
+                        //TODO consolidate catalogue and MeasureReport group names, also between projects
+                        let group_key = if stratifier_key == "diagnosis" {
+                            "diagnosis"
+                        } else {
+                            &group_key
+                        };
+                        let criteria_counts_maybe =
+                            get_stratifier(counts, &group_key, &stratifier_key);
+                        if let Some(criteria_counts) = criteria_counts_maybe {
+                            for criterion_count in criteria_counts {
+                                let (key, count) = (criterion_count.0, criterion_count.1);
+                                let catalogue_criterion = CatalogueCriterion {
+                                    key: key.clone(),
+                                    name: key.clone(),
+                                    description: key.clone(),
+                                    count: count.clone(),
+                                };
+                                criteria.push(json!(catalogue_criterion));
                             }
-                        }                        
+                        }
                     }
                 }
             }
-        },
+        }
         _ => {}
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    const CATALOGUE_BBMRI_NO_DIAGNOSES: &str =
+        include_str!("resources/test/catalogue_bbmri_no_diagnoses.json");
+    const CATALOGUE_BBMRI: &str = include_str!("resources/test/catalogue_bbmri.json");
+    const CRITERIA_GROUPS_BBMRI: &str = include_str!("resources/test/criteria_groups_bbmri.json");
+
+    #[test]
+    fn test_recurse_bbmri() {
+        let mut criteria_groups: CriteriaGroups =
+            serde_json::from_str(&CRITERIA_GROUPS_BBMRI).expect("Not valid criteria groups");
+
+        let mut catalogue =
+            serde_json::from_str(&CATALOGUE_BBMRI_NO_DIAGNOSES).expect("Not valid json");
+
+        recurse(&mut catalogue, &mut criteria_groups);
+
+        pretty_assertions::assert_eq!(
+            CATALOGUE_BBMRI,
+            serde_json::to_string(&catalogue).expect("Failed to serialize JSON")
+        );
+    }
+}

src/resources/test/catalogue_bbmri.json

@@ -0,0 +1 @@
+[{"childCategories":[{"criteria":[{"count":43,"description":"male","key":"male","name":"male"},{"count":31,"description":"female","key":"female","name":"female"},{"description":"other","key":"other","name":"other"},{"description":"unknown","key":"sex_uncharted","name":"unknown"}],"fieldType":"single-select","infoButtonText":["The gender search criterion represents the administrative gender of the patient."],"key":"gender","name":"Gender","system":"","type":"EQUALS"},{"criteria":[{"count":26,"description":"C34.0","key":"C34.0","name":"C34.0"},{"count":28,"description":"C34.2","key":"C34.2","name":"C34.2"},{"count":25,"description":"C34.8","key":"C34.8","name":"C34.8"},{"count":27,"description":"C78.0","key":"C78.0","name":"C78.0"},{"count":25,"description":"D38.6","key":"D38.6","name":"D38.6"},{"count":25,"description":"R91","key":"R91","name":"R91"}],"fieldType":"autocomplete","infoButtonText":["If diagnosis is selected, the search returns all specimens from all patients who have at least one Condition resource OR Specimen resource with one of the selected diagnoses."],"key":"diagnosis","name":"Diagnosis ICD-10","system":"http://fhir.de/CodeSystem/dimdi/icd-10-gm","type":"EQUALS"},{"criteria":[],"fieldType":"number","infoButtonText":["The donor age at diagnosis criterion represents the age of the patient at the time of diagnosis, calculated as the difference between the date of disease onset and the patient's date of birth. That's why the age stratifier, which shows today's age, also shows some ages that are outside the selected range."],"key":"diagnosis_age_donor","name":"Diagnosis age donor (years)","system":"","type":"BETWEEN"},{"criteria":[],"fieldType":"date","infoButtonText":["Date of diagnosis represents the date when the donor was diagnosed."],"key":"date_of_diagnosis","name":"Date of diagnosis","system":"","type":"BETWEEN"}],"key":"donor","name":"Donor/Clinical Information"},{"childCategories":[{"criteria":[],"fieldType":"number","infoButtonText":["The donor age criterion represents the patient's age today"],"key":"donor_age","name":"Donor Age","system":"","type":"BETWEEN"},{"criteria":[{"count":62,"description":"Serum","key":"blood-serum","name":"Serum"},{"description":"Tissue snap frozen","key":"tissue-frozen","name":"Tissue snap frozen"},{"description":"Whole Blood","key":"whole-blood","name":"Whole Blood"},{"count":62,"description":"Plasma","key":"blood-plasma","name":"Plasma"},{"description":"Other derivative","key":"derivative-other","name":"Other derivative"},{"description":"Other tissue storage","key":"tissue-other","name":"Other tissue storage"},{"description":"Peripheral blood cells","key":"peripheral-blood-cells-vital","name":"Peripheral blood cells"},{"description":"Urine","key":"urine","name":"Urine"},{"description":"RNA","key":"rna","name":"RNA"},{"description":"Other liquid biosample","key":"liquid-other","name":"Other liquid biosample"},{"description":"Buffy coat","key":"buffy-coat","name":"Buffy coat"},{"description":"DNA","key":"dna","name":"DNA"},{"description":"Liquor/CSF","key":"csf-liquor","name":"Liquor/CSF"},{"description":"Faeces","key":"stool-faeces","name":"Faeces"},{"description":"Bone marrow","key":"bone-marrow","name":"Bone marrow"},{"description":"Tissue (FFPE)","key":"tissue-ffpe","name":"Tissue (FFPE)"},{"description":"Saliva","key":"saliva","name":"Saliva"},{"description":"Ascites","key":"ascites","name":"Ascites"},{"description":"Swab","key":"swab","name":"Swab"},{"description":"Dried whole blood","key":"dried-whole-blood","name":"Dried whole blood"}],"fieldType":"single-select","infoButtonText":["If sample types are selected, only samples of those types will be displayed"],"key":"sample_kind","name":"Sample type","system":"","type":"EQUALS"},{"criteria":[],"fieldType":"date","infoButtonText":["If a sampling date range is selected, only those samples collected within that date range will be displayed. Samples that don't have this information will not be displayed."],"key":"sampling_date","name":"Sampling date","system":"","type":"BETWEEN"},{"criteria":[{"description":"RT","key":"temperatureRoom","name":"RT"},{"description":"2°C to 10°C","key":"temperature2to10","name":"2°C to 10°C"},{"description":"4°C","key":"four_degrees","name":"4°C"},{"description":"Minus 18°C to minus 35°C","key":"temperature-18to-35","name":"Minus 18°C to minus 35°C"},{"description":"Minus 60°C to minus 85°C","key":"temperature-60to-85","name":"Minus 60°C to minus 85°C"},{"description":"Gaseous nitrogen","key":"temperatureGN","name":"Gaseous nitrogen"},{"description":"Liquid nitrogen","key":"temperatureLN","name":"Liquid nitrogen"},{"description":"Other storage temperature","key":"temperatureOther","name":"Other storage temperature"},{"description":"Uncharted storage temperature","key":"storage_temperature_uncharted","name":"Uncharted storage temperature"}],"fieldType":"single-select","infoButtonText":["If storage temperatures are selected, only those samples stored at one of those temperatures will be displayed. Samples that don't have this information will not be displayed."],"key":"storage_temperature","name":"Storage temperature","system":"","type":"EQUALS"}],"key":"sample","name":"Sample"}]