Merge pull request #11 from UNSW-CEEM/event_time_updates

Allow user to provide exact event time and not need to change it for each dataset

island_assessment/island_assessment_functions.R

@@ -2,7 +2,7 @@ classify_islands <- function(combined_data, alert_data, event_time, window_lengt
   # add col to circ_sum with islanded flags, also for freqwobble and undervoltage. 
   combined_data <- identify_islanded_sites(combined_data, alert_data, event_time)
   # determine which islanded sites can be classified as disconnect
-  event_window_data <- filter(combined_data, ts >= event_time & ts <= event_time + 60 * window_length)
+  event_window_data <- filter(combined_data, ts > event_time - d & ts <= event_time + 60 * window_length)
   event_window_data <- assess_islands(event_window_data)
   combined_data <- left_join(combined_data, event_window_data, on=c("c_id", "clean"))
   # replace response_category with island_assessment response

island_assessment/tests/test_island_assessment.R

@@ -254,11 +254,12 @@ test_that("Test high level classify_islands fn" ,{
 
   c_id <- c(3000)
   ts <- as.POSIXct("2021-01-24 16:47:00", tz = "Australia/Brisbane")
+  d <- 60
   clean <- c(1)
   response_category <- c('3 Drop to Zero')
   f <- c(52)
   v <- c(240)
-  combined_data <- data.frame(c_id, ts, clean, response_category, f, v, stringsAsFactors = FALSE)
+  combined_data <- data.frame(c_id, ts, d, clean, response_category, f, v, stringsAsFactors = FALSE)
 
   first_timestamp <- c(1611470760257)
   GridFaultContactorTrip <- c(1)
@@ -274,7 +275,7 @@ test_that("Test high level classify_islands fn" ,{
   Islanded <- c(1)
   island_assessment <- c('Frequency disruption')
   islanding_alert <- c('Islanded - Freq Wobble')
-  expected_output <- data.frame(c_id, ts, clean, response_category, f, v, Islanded,
+  expected_output <- data.frame(c_id, ts, d, clean, response_category, f, v, Islanded,
                                 island_assessment, islanding_alert, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
 })
@@ -284,11 +285,12 @@ test_that("Test high level classify_islands fn for a non-islanded site" ,{
 
   c_id <- c(3000)
   ts <- as.POSIXct("2021-01-24 16:47:00", tz = "Australia/Brisbane")
+  d <- 60
   clean <- c(1)
   response_category <- c('3 Drop to Zero')
   f <- c(52)
   v <- c(240)
-  combined_data <- data.frame(c_id, ts, clean, response_category, f, v, stringsAsFactors = FALSE)
+  combined_data <- data.frame(c_id, ts, d, clean, response_category, f, v, stringsAsFactors = FALSE)
 
   first_timestamp <- c(1611470760257)
   GridFaultContactorTrip <- c(0)
@@ -304,7 +306,7 @@ test_that("Test high level classify_islands fn for a non-islanded site" ,{
   Islanded <- c(0)
   island_assessment <- c(NA_character_)
   islanding_alert <- c("NA")
-  expected_output <- data.frame(c_id, ts, clean, response_category, f, v, Islanded,
+  expected_output <- data.frame(c_id, ts, d, clean, response_category, f, v, Islanded,
                                 island_assessment, islanding_alert, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)
 })

normalised_power/normalised_power_function.R

@@ -1,5 +1,5 @@
 event_normalised_power <- function(combined_data, event_time, keep_site_id){
-  event_time_data <- data.frame(filter(combined_data, Time == event_time))
+  event_time_data <- data.frame(filter(combined_data, Time > event_time - d & Time <= event_time))
   event_time_data <- setnames(event_time_data, c("site_performance_factor"), c("event_site_performance_factor"))
   if (keep_site_id){
     event_time_data <- select(event_time_data, site_id, event_site_performance_factor)

response_categorisation/response_categorisation_function.R

@@ -1,14 +1,15 @@
 # if first ts != event_time -> NA (also lets count how many lose to this)
 
 categorise_response <- function(combined_data, event_time, window_length, NED_threshold_pct = 0.8){
-  event_window_data <- filter(combined_data, ts >= event_time & ts <= event_time + 60 * window_length)
+  event_window_data <- filter(combined_data, ts > event_time - d & ts <= event_time + 60 * window_length)
   event_window_data <- event_window_data[order(event_window_data$ts),]
   event_window_data <- group_by(event_window_data, c_id, clean)
   event_window_data <- summarise(event_window_data, d=first(d), event_power=first(power_kW),
                                  min_norm_power=min(power_kW/first(power_kW)),
                                  num_con_zeros=num_consecutive_zeros(power_kW),
                                  num_data_points=length(power_kW)-1,
-                                 first_ts = ifelse(first(ts)==event_time,"sampled","not sampled"))
+                                 first_ts = ifelse(first(ts) > event_time - d & first(ts) <= event_time,
+                                                   "sampled", "not sampled"))
   event_window_data <- categorise_by_response(event_window_data, window_length, NED_threshold_pct)
   event_window_data <- select(event_window_data, "c_id", "clean", "response_category")
   combined_data <- left_join(combined_data, event_window_data, on=c("c_id", "clean"))

run_analysis/run_analysis.R

@@ -18,7 +18,7 @@ validate_pre_event_interval <- function(pre_event_interval, load_start_time, loa
   start_time <- as.POSIXct(load_start_time, tz = "Australia/Brisbane")
   end_time <- as.POSIXct(load_end_time, tz = "Australia/Brisbane")
   pre_event_interval <- as.POSIXct(pre_event_interval, tz = "Australia/Brisbane")
-  data_at_set_pre_event_interval = filter(data, ts == pre_event_interval)
+  data_at_set_pre_event_interval = filter(data, ts > pre_event_interval - d & ts <= pre_event_interval)
   if (dim(data_at_set_pre_event_interval)[1] == 0) {
     long_error_message <- c("The pre-event time interval does not match any time step in the time series data.")
     long_error_message <- paste(long_error_message, collapse = '')
@@ -104,7 +104,7 @@ ufls_detection <- function(
   combined_data_f <- mutate(combined_data_f, response_category =
                             if_else((ufls_status == "UFLS Dropout") |
                                       (ufls_status_v == "UFLS Dropout"),
-                                    "UFLS Dropout", response_category))
+                                    "UFLS Dropout", response_category, missing=response_category))
   return(combined_data_f)
 }
 
@@ -130,7 +130,7 @@ determine_distance_zones <- function(
 #' @return An updated dataframe with the column c_id_norm_power added
 normalise_c_id_power_by_pre_event <- function(combined_data_f, pre_event_interval) {
   logdebug('Calc event normalised power', logger=logger)
-  event_powers <- filter(combined_data_f, ts == pre_event_interval)
+  event_powers <- filter(combined_data_f, ts > pre_event_interval - d & ts <= pre_event_interval)
   event_powers <- mutate(event_powers, event_power=power_kW)
   event_powers <- select(event_powers, c_id, clean, event_power)
   combined_data_f <- left_join(combined_data_f, event_powers, by=c("c_id", "clean"))
@@ -146,7 +146,7 @@ normalise_c_id_power_by_daily_max <- function(combined_data, max_power, pre_even
   combined_data_f <- left_join(combined_data, max_power, by=c("c_id", "clean"))
   combined_data_f <- mutate(combined_data_f, c_id_daily_norm_power=power_kW/max_power)
   if (!missing(pre_event_interval)){
-    pre_event_daily_norm_power <- filter(combined_data_f, ts == pre_event_interval)
+    pre_event_daily_norm_power <- filter(combined_data_f, ts > pre_event_interval - d & ts <= pre_event_interval)
     pre_event_daily_norm_power <- mutate(pre_event_daily_norm_power, pre_event_norm_power = c_id_daily_norm_power)
     pre_event_daily_norm_power <- select(pre_event_daily_norm_power, clean, c_id, pre_event_norm_power)
     combined_data_f <- left_join(combined_data_f, pre_event_daily_norm_power, by=c("c_id", "clean"))
@@ -311,7 +311,7 @@ run_analysis <- function(data, settings) {
       combined_data_f <- normalise_c_id_power_by_daily_max(
         combined_data_f, max_power, settings$pre_event_interval
       )
-      event_window_data <- filter(combined_data_f, ts >= settings$pre_event_interval)
+      event_window_data <- filter(combined_data_f, ts > settings$pre_event_interval - d)
       reconnection_categories <- create_reconnection_summary(event_window_data, settings$pre_event_interval,
                                                               settings$disconnecting_threshold,
                                                               reconnect_threshold = settings$reconnection_threshold,
@@ -406,12 +406,12 @@ run_analysis <- function(data, settings) {
 
         # Combine data sets that have the same grouping so they can be saved in a single file
         if (no_grouping){
-          et <- settings$pre_event_interval
+          #et <- settings$pre_event_interval
           # agg_norm_power <- event_normalised_power(agg_norm_power, et, keep_site_id=TRUE)
           data$agg_power <- left_join(data$agg_power, data$agg_norm_power[, c("c_id_norm_power", "c_id", "Time")], 
                                     by=c("Time", "c_id"))
         } else {
-          et <- settings$pre_event_interval
+          #et <- settings$pre_event_interval
           # agg_norm_power <- event_normalised_power(agg_norm_power, et,  keep_site_id=FALSE)
           data$agg_power <- left_join(data$agg_power, data$agg_norm_power[, c("c_id_norm_power", "series", "Time")], 
                                     by=c("Time", "series"))

shiny.R

@@ -675,7 +675,7 @@ server <- function(input,output,session){
                   value=strftime(floor_date(get_mode(v$combined_data$ts), "day"), format="%Y-%m-%d"), startview="year")
       })
       output$pre_event_interval <- renderUI({
-        timeInput("pre_event_interval", label=strong('Pre-event time interval (Needs to match exactly to data timestamp)'), 
+        timeInput("pre_event_interval", label=strong('Pre-event time interval'), 
                   value = as.POSIXct("12:13:55",format="%H:%M:%S"))
       })
       output$window_length <- renderUI({

ufls_detection/tests/test_ufls_detection_voltage.R

@@ -2,7 +2,7 @@ context("Testing the DER event analysis UFLS detection based on voltage")
 
 # Define constants
 window_length <- 5
-event_time <- "2018-01-01 13:11:55"
+event_time <- "2018-01-01 13:11:50"
 event_time <- as.POSIXct(strptime(event_time, "%Y-%m-%d %H:%M:%S", 
                                   tz="Australia/Brisbane"))
 ts_3_times <- c("2018-01-01 13:10:55", "2018-01-01 13:11:55", 
@@ -70,7 +70,7 @@ test_that("Test a UFLS example with no pre-event timesteps", {
   out <- ufls_detection_voltage(input_data, event_time, window_length)
   c_id <- c(1)
   pre_event_v_mean <- c(NA_integer_)
-  post_event_v_mean <- c(240)
+  post_event_v_mean <- c(239)
   ufls_status_v <- c(NA_character_)
   expected_output <- data.frame(c_id, pre_event_v_mean, post_event_v_mean, ufls_status_v, stringsAsFactors = FALSE)
   expect_equal(out, expected_output, tolerance=0.001)

ufls_detection/ufls_detection_voltage.R

@@ -4,8 +4,8 @@ ufls_detection_voltage <- function(combined_data, event_time, window_length,
   pre_event_window_start <- event_time - 60 * window_length
   post_event_window_end <- event_time + 60 * window_length
   pre_event_window <- filter(combined_data, 
-                             ts >= pre_event_window_start, 
-                             ts < event_time)
+                             ts > pre_event_window_start, 
+                             ts <= event_time)
   post_event_window <- filter(combined_data, 
                               ts > event_time + post_event_delay, 
                               ts <= post_event_window_end + post_event_delay)