Merge pull request #26 from UNSW-CEEM/upgrade-installation

Upgrade installation. 

Use the latest versions of dependencies instead of relying upon outdated versions. This should solve the issues with installation currently where outdated versions dependencies are unable to be found and installed.

By updating to the latest versions of packages, there are some minor breakages due to changes in dependencies. Update the code to work with the latest versions of dependencies.

All updates to the code should also be backwards compatible. Tested with R versions 3.5.2, 4.2.2 and the latest version 4.3.1.

.github/workflows/r_tests.yml

@@ -21,12 +21,12 @@ permissions:
 
 jobs:
   test:
-    name: Run tests on ${{ matrix.os }}
+    name: Run tests on ${{ matrix.os }} for R version ${{ matrix.r-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: ['macos-latest', 'windows-latest']
-        r-version: ['3.6.3']
+        r-version: ['4.2.2', 'latest']
 
     steps:
       - uses: actions/checkout@v3

BDInterface/data_cleaning_functions.R

@@ -49,7 +49,7 @@ site_details_data_cleaning_one <- function(site_details){
 
 site_details_data_cleaning_two <- function(time_series, site_details){
   max_site_power <- calc_max_kw_per_site(time_series)
-  site_details <- left_join(site_details, max_site_power, on=c("site_id"))
+  site_details <- left_join(site_details, max_site_power, by=c("site_id"))
   site_details <- check_for_peak_power_greater_than_dc_capacity(site_details)
   # Record if dc value was scaled or not.
   site_details <- mutate(site_details, change_dc=ifelse(sum_dc!=sum_dc_old,1,0))
@@ -113,7 +113,7 @@ clean_connection_types <- function(combined_data, circuit_details, postcode_data
   # Select the values from the orginal circuit details that would not be changed 
   # by cleaning, then merge back in with details updated or created by cleaning
   circuit_details <- select(circuit_details, site_id, c_id, manual_droop_compliance, manual_reconnect_compliance)
-  combined_data <- left_join(combined_data, circuit_details, on="c_id")
+  combined_data <- left_join(combined_data, circuit_details, by="c_id")
   return(combined_data)
 }
 

aggregation/aggregate_functions.R

@@ -83,7 +83,7 @@ vector_groupby_count_zones <- function(data, grouping_cols){
   if (!"zone" %in% grouping_cols) {grouping_cols <- c(grouping_cols, "zone")}
   grouping_cols <- grouping_cols[grouping_cols != "c_id"]
   add_cols <- grouping_cols[!grouping_cols %in% c("clean", "zone")]
-  grouping_cols <- c(grouping_cols, add_cols)
+  grouping_cols <- unique(c(grouping_cols, add_cols))
   data <- group_by(data, .dots=grouping_cols)
   data <- summarise(data , sample_count=length(unique(c_id)))
   data$series_x <- do.call(paste, c(data[c("clean", "zone")], sep = "-" ))

install.R

@@ -1,31 +1,41 @@
-install.packages("versions")
-library("versions")
-install_type <- 'binary'
-install.versions(c('shiny'), c('1.4.0.2'), type=install_type)
-install.versions(c('shinyalert'), c('2.0.0'), type=install_type)
-install.versions(c('shinyTime'), c('1.0.1'), type=install_type)
-install.versions(c('shinyWidgets'), c('0.5.4'), type=install_type)
-install.versions(c('plotly'), c('4.9.2.1'), type=install_type)
-install.versions(c('lubridate'), c('1.7.8'), type=install_type)
-install.versions(c('dplyr'), c('0.8.5'), type=install_type)
-install.versions(c('tidyr'), c('1.0.2'), type=install_type)
-install.versions(c('data.table'), c('1.12.8'), type=install_type)
-install.versions(c('shinycssloaders'), c('1.0.0'), type=install_type)
-install.versions(c('shinyFiles'), c('0.8.0'), type=install_type)
-install.versions(c('shinyjs'), c('2.0.0'), type=install_type)
-install.versions(c('fasttime'), c('1.0-2'), type=install_type)
-install.versions(c('DT'), c('0.16'), type=install_type)
-install.versions(c('suncalc'), c('0.5.0'), type=install_type)
-install.versions(c('ggmap'), c('3.0.0'), type=install_type)
-install.versions(c('measurements'), c('1.4.0'), type=install_type)
-install.versions(c('assertthat'), c('0.2.1'), type=install_type)
-install.versions(c('geosphere'), c('1.5-10'), type=install_type)
-install.versions(c('swfscMisc'), c('1.3'), type=install_type)
-install.versions(c('padr'), c('0.5.1'), type=install_type)
-install.versions(c('sqldf'), c('0.4-11'), type=install_type)
-install.versions(c('gridExtra'), c('2.3'), type=install_type)
-install.versions(c('rjson'), c('0.2.20'), type=install_type)
-install.versions(c('R6'), c('2.5.0'), type=install_type)
-install.versions(c('git2r'), c('0.29.0'), type=install_type)
-install.versions(c('logging'), c('0.10-108'), type=install_type)
-install.versions(c('testthat'), c('2.3.2'), type=install_type)
+#' Load libraries
+#'
+#' using tries to load all libraries passed to it, and if a library is not
+#' found, using will install it and then load it.
+#' @param ... The libraries / dependencies / packages to load.
+using <- function(...) {
+    libs <- unlist(list(...))
+    req <- unlist(lapply(libs, require, character.only = TRUE))
+    need <- libs[req == FALSE]
+    if (length(need) > 0) {
+        install.packages(need)
+        lapply(need, require, character.only = TRUE)
+    }
+}
+
+using(
+    "plotly",
+    "lubridate",
+    "dplyr",
+    "tidyr",
+    "data.table",
+    "shinyFiles",
+    "shinyjs",
+    "fasttime",
+    "DT",
+    "suncalc",
+    "assertthat",
+    "geosphere",
+    "swfscMisc",
+    "padr",
+    "sqldf",
+    "rjson",
+    "R6",
+    "git2r",
+    "logging",
+    "testthat",
+    "shiny",
+    "shinyTime",
+    "shinyWidgets",
+    "shinyalert"
+)

island_assessment/island_assessment_functions.R

@@ -4,7 +4,7 @@ classify_islands <- function(combined_data, alert_data, event_time, window_lengt
   # determine which islanded sites can be classified as disconnect
   event_window_data <- filter(combined_data, ts > event_time - d & ts <= event_time + 60 * window_length)
   event_window_data <- assess_islands(event_window_data)
-  combined_data <- left_join(combined_data, event_window_data, on=c("c_id", "clean"))
+  combined_data <- left_join(combined_data, event_window_data, by=c("c_id", "clean"))
   # replace response_category with island_assessment response
   combined_data <- replace_response_with_alert(combined_data)
   return(combined_data)

island_assessment/tests/test_island_assessment.R

@@ -106,7 +106,7 @@ test_that("Test assess_islands with response_category = 'NED' and v, f are norma
   v <- c(240)
   combined_data <- data.frame(c_id, ts, clean, Islanded, response_category, f, v, stringsAsFactors = FALSE)
 
-  out <- assess_islands(combined_data)
+  out <- as.data.frame(assess_islands(combined_data))
   island_assessment <- c("Otherwise normal")
   expected_output <- data.frame(c_id, clean, island_assessment, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
@@ -123,7 +123,7 @@ test_that("Test assess_islands with max_freq = 55Hz" ,{
   v <- c(240)
   combined_data <- data.frame(c_id, ts, clean, Islanded, response_category, f, v, stringsAsFactors = FALSE)
 
-  out <- assess_islands(combined_data)
+  out <- as.data.frame(assess_islands(combined_data))
   island_assessment <- c('Gateway curtailed')
   expected_output <- data.frame(c_id, clean, island_assessment, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
@@ -140,7 +140,7 @@ test_that("Test assess_islands with min_freq = 49Hz" ,{
   v <- c(240)
   combined_data <- data.frame(c_id, ts, clean, Islanded, response_category, f, v, stringsAsFactors = FALSE)
 
-  out <- assess_islands(combined_data)
+  out <- as.data.frame(assess_islands(combined_data))
   island_assessment <- c('Frequency disruption')
   expected_output <- data.frame(c_id, clean, island_assessment, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
@@ -158,7 +158,7 @@ test_that("Test assess_islands with fmin = 48Hz but minimum f = 50 Hz" ,{
   v <- c(240)
   combined_data <- data.frame(c_id, ts, clean, Islanded, response_category, f, fmin, v, stringsAsFactors = FALSE)
 
-  out <- assess_islands(combined_data)
+  out <- as.data.frame(assess_islands(combined_data))
   island_assessment <- c('Frequency disruption')
   expected_output <- data.frame(c_id, clean, island_assessment, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
@@ -175,7 +175,7 @@ test_that("Test assess_islands with max_voltage = 270Hz" ,{
   v <- c(270)
   combined_data <- data.frame(c_id, ts, clean, Islanded, response_category, f, v, stringsAsFactors = FALSE)
 
-  out <- assess_islands(combined_data)
+  out <- as.data.frame(assess_islands(combined_data))
   island_assessment <- c('Voltage disruption')
   expected_output <- data.frame(c_id, clean, island_assessment, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
@@ -192,7 +192,7 @@ test_that("Test assess_islands for site with no other disruptions" ,{
   v <- c(241)
   combined_data <- data.frame(c_id, ts, clean, Islanded, response_category, f, v, stringsAsFactors = FALSE)
 
-  out <- assess_islands(combined_data)
+  out <- as.data.frame(assess_islands(combined_data))
   island_assessment <- c('PV disconnect')
   expected_output <- data.frame(c_id, clean, island_assessment, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)

load_tool_environment.R

@@ -1,29 +1,29 @@
-library(shiny)
-library(shinyTime)
-library(shinyWidgets)
-library(shinyalert)
 library(plotly)
 library(lubridate)
 library(dplyr)
 library(tidyr)
 library(data.table)
-library(shinycssloaders)
+## library(shinycssloaders)
 library(shinyFiles)
 library(shinyjs)
-library(stringr)
+## library(stringr)
 library(fasttime)
 library(DT)
 library(suncalc)
-library(ggmap)
-library(measurements)
+## library(ggmap)
+## library(measurements)
 library(assertthat)
 library(geosphere)
 library(swfscMisc)
 library(padr)
 library(sqldf)
-library(gridExtra)
+## library(gridExtra)
 library(rjson)
 library(logging)
+library(shiny)
+library(shinyTime)
+library(shinyWidgets)
+library(shinyalert)
 
 source("process_input_data/process_input_data_functions.R")
 source("aggregation/aggregate_functions.R")

process_input_data/process_input_data_functions.R

@@ -66,24 +66,27 @@ sum_manufacturers <- function(manufacturers){
   return(manufacturer)
 }
 
-assert_raw_site_details_assumptions <- function(site_details){
-  # Check in coming site data for conformance to data processing assumptions
-  # We assume that only possible s_state values are NSW, QLD, VIC, TAS, SA, WA, NT, ACT
-  s_state <- site_details$s_state
-  assert_that(all(s_state == "NSW" | s_state == "QLD" | s_state == "VIC" | s_state == "TAS" | s_state == "SA" |
-                    s_state == "WA" | s_state == "ACT"), msg="State values outside expected set NSW, ACT, SA etc")
-  # We assume that for each site id there is only one distinct s_state value  and s_postcode value
-  site_details_grouped <- group_by(site_details, site_id)
-  site_details_grouped <- summarise(site_details_grouped, s_state=unique(s_state), s_postcode=unique(s_postcode))
-  site_details_grouped <- as.data.frame(site_details_grouped)
-  assert_that(all(lapply(site_details_grouped$s_state, length)==1), 
-              msg="Some sites have mutiple distinct s_state values")
-  assert_that(all(lapply(site_details_grouped$s_postcode, length)==1),
-              msg="Some sites have mutiple distinct s_postcode values")
-  # We assume ac and dc values can be converted to numeric without be turned
-  # into NAs
-  assert_that(all(!is.na(as.numeric(site_details$ac))))
-  #assert_that(all(!is.na(as.numeric(site_details$dc))))
+#' Check incoming site data for conformance to data processing assumptions.
+assert_raw_site_details_assumptions <- function(site_details) {
+    # Only possible s_state values are NSW, QLD, VIC, TAS, SA, WA, NT, ACT.
+    s_state <- site_details$s_state
+    assert_that(
+        all(s_state %in% c("NSW", "QLD", "VIC", "TAS", "SA" , "WA", "ACT")),
+        msg = "State values outside expected set NSW, ACT, SA etc."
+    )
+    # Only one distinct s_state and s_postcode value for each site_id.
+    site_details_grouped <- group_by(site_details, site_id) %>%
+        summarise(s_state = unique(s_state), s_postcode = unique(s_postcode))
+    assert_that(
+        all(count(site_details_grouped, site_id, s_state)$n == 1),
+        msg = "Some sites have mutiple distinct s_state values."
+    )
+    assert_that(
+        all(count(site_details_grouped, site_id, s_postcode)$n == 1),
+        msg = "Some sites have mutiple distinct s_postcode values."
+    )
+    # Assume AC values can be converted to numeric without be turned into NAs.
+    assert_that(all(!is.na(as.numeric(site_details$ac))))
 }
 
 perform_power_calculations <- function(master_data_table){

reconnect_compliance/tests/test_categorise_reconnection_compliance.R

@@ -41,5 +41,5 @@ testthat::test_that("Calculating categorisation works",{
                              6,                  Cannot-be-set"
 
   expected_categories <- load_test_df_keep_NA(expected_categories)
-  testthat::expect_equal(reconnection_times, expected_categories, tolerance = 1e-4)
-})
+  testthat::expect_equivalent(reconnection_times, expected_categories, tolerance = 1e-4)
+})

reconnect_compliance/tests/test_create_reconnection_summary.R

@@ -44,7 +44,7 @@ testthat::test_that("Categorising a circuit with 5s data, and just a breif ramp
                                                     ramp_rate_change_resource_limit_threshold = -0.1)
 
 
-  testthat::expect_equal(calculated_results, expected_results, tolerance = 1e-4)
+  testthat::expect_equivalent(calculated_results, expected_results, tolerance = 1e-4)
 })
 
 
@@ -82,7 +82,7 @@ testthat::test_that("Categorising a circuit with 5s data, sustained ramp rate vi
                                                     ramp_threshold_for_non_compliance = 0.3,
                                                     ramp_rate_change_resource_limit_threshold = -0.1)
 
-  testthat::expect_equal(calculated_results, expected_results, tolerance = 1e-4)
+  testthat::expect_equivalent(calculated_results, expected_results, tolerance = 1e-4)
 })
 
 
@@ -120,7 +120,7 @@ testthat::test_that("Categorising a circuit with 60s data, no ramp rate violatio
                                                     ramp_threshold_for_non_compliance = 0.3,
                                                     ramp_rate_change_resource_limit_threshold = -0.1)
 
-  testthat::expect_equal(calculated_results, expected_results, tolerance = 1e-4)
+  testthat::expect_equivalent(calculated_results, expected_results, tolerance = 1e-4)
 })
 
 
@@ -159,5 +159,5 @@ testthat::test_that("Categorising a circuit with 5s data, sustained ramp rate vi
                                                     ramp_rate_change_resource_limit_threshold = -0.1)
 
 
-  testthat::expect_equal(calculated_results, expected_results, tolerance = 1e-4)
-})
+  testthat::expect_equivalent(calculated_results, expected_results, tolerance = 1e-4)
+})

reconnect_compliance/tests/test_reconnection_time_calculation.R

@@ -46,5 +46,5 @@ testthat::test_that("Calculating reconnection times works",{
   calculated_reconnection_times <- calculate_reconnection_times(normalised_power, event_time, 
                                                                 disconnect_threshold = 0.05, reconnect_threshold = 0.95)
 
-  testthat::expect_equal(calculated_reconnection_times, expected_reconnection_times, tolerance = 1e-4)
-})
+  testthat::expect_equivalent(calculated_reconnection_times, expected_reconnection_times, tolerance = 1e-4)
+})

response_categorisation/response_categorisation_function.R

@@ -12,7 +12,7 @@ categorise_response <- function(combined_data, event_time, window_length, NED_th
                                                    "sampled", "not sampled"))
   event_window_data <- categorise_by_response(event_window_data, window_length, NED_threshold_pct)
   event_window_data <- select(event_window_data, "c_id", "clean", "response_category")
-  combined_data <- left_join(combined_data, event_window_data, on=c("c_id", "clean"))
+  combined_data <- left_join(combined_data, event_window_data, by=c("c_id", "clean"))
   return(combined_data)
 }
 

upscale_disconnections/tests/test_summarise_disconnections.R

@@ -31,7 +31,7 @@ testthat::test_that("group_disconnections_by_manufacturer",{
   circuit_summary <- load_test_file(circuit_summary)
   expected_output <- load_test_file(expected_output)
   output <- group_disconnections_by_manufacturer(circuit_summary)
-  testthat::expect_equal(output, expected_output, tolerance = 1e-4)
+  testthat::expect_equivalent(output, expected_output, tolerance = 1e-4)
 })
 
 testthat::test_that("join_circuit_summary_and_cer_manufacturer_data",{

upscale_disconnections/tests/test_summarise_disconnections_excluding_ufls.R

@@ -31,7 +31,7 @@ testthat::test_that("group_disconnections_by_manufacturer",{
   circuit_summary <- load_test_file(circuit_summary)
   expected_output <- load_test_file(expected_output)
   output <- group_disconnections_by_manufacturer(circuit_summary, exclude_ufls_circuits = TRUE)
-  testthat::expect_equal(output, expected_output, tolerance = 1e-4)
+  testthat::expect_equivalent(output, expected_output, tolerance = 1e-4)
 })
 
 testthat::test_that("get_number_of_ufls_disconnections",{

upscaling/upscale_function.R

@@ -21,7 +21,7 @@ calc_site_performance_factors <- function(performance_data){
   performace_data_p <- group_by(performance_data, ts, site_id, clean)
   performace_data_p <- summarise(performace_data_p , site_performance_factor=sum(power_kW))
   performace_data_p <- as.data.frame(performace_data_p)
-  performance_data <- left_join(performance_data, performace_data_p, on=c('site_id', 'ts', 'clean'))
+  performance_data <- left_join(performance_data, performace_data_p, by=c('site_id', 'ts', 'clean'))
   performance_data <- mutate(performance_data, site_performance_factor=(site_performance_factor/sum_ac))
   return(performance_data)
 }

validation/validate_results.R

@@ -142,6 +142,9 @@ compare_dfs <- function(reference, test, event_name=NA){
     # 1. check for new columns
     ref_columns <- names(reference)
     test_columns <- names(test)
+    # ignore tool_hash, since it will always be different
+    ref_columns <- ref_columns[!ref_columns %in% c("tool_hash")]
+    test_columns <- test_columns[!test_columns %in% c("tool_hash")]
     column_diff <- difference_between_lists(ref_columns, test_columns)
 
     if (length(column_diff$reference_only) > 0) {