diff --git a/modules/audio_output/coreaudio_common.c b/modules/audio_output/coreaudio_common.c
index e202530af70f..36fdd2ca78fc 100644
--- a/modules/audio_output/coreaudio_common.c
+++ b/modules/audio_output/coreaudio_common.c
@@ -146,24 +146,7 @@ GetLatency(audio_output_t *p_aout)
                                    : p_sys->i_dev_latency_ticks;
 
     /* Add the AudioUnit latency to the auhal/audiounit_ios latency */
-    if (p_sys->au != NULL)
-    {
-        Float64 unit_s;
-        if (AudioUnitGetProperty(p_sys->au, kAudioUnitProperty_Latency,
-                                 kAudioUnitScope_Global, 0, &unit_s,
-                                 &(UInt32) { sizeof(unit_s) }) != noErr)
-            unit_s = 0;
-        vlc_tick_t us = vlc_tick_from_sec(unit_s);
-        if (us != p_sys->au_latency_ticks)
-        {
-            msg_Dbg(p_aout, "Adding AudioUnit latency: %" PRId64 "us", us);
-            p_sys->au_latency_ticks = us;
-        }
-
-        dev_latency_ticks += us;
-    }
-
-    return dev_latency_ticks;
+    return dev_latency_ticks + p_sys->au_latency_ticks;
 }
 
 /* Called from render callbacks. No lock, wait, and IO here */
@@ -361,7 +344,6 @@ ca_Initialize(audio_output_t *p_aout, const audio_sample_format_t *fmt,
 {
     struct aout_sys_common *p_sys = (struct aout_sys_common *) p_aout->sys;
 
-    p_sys->au = NULL;
     p_sys->au_latency_ticks = 0;
     p_sys->i_underrun_size = 0;
     p_sys->b_paused = false;
@@ -874,7 +856,15 @@ au_Initialize(audio_output_t *p_aout, AudioUnit au, audio_sample_format_t *fmt,
         AudioUnitUninitialize(au);
         return VLC_EGENERIC;
     }
-    p_sys->au = au;
+
+    Float64 unit_s;
+    if (AudioUnitGetProperty(au, kAudioUnitProperty_Latency,
+                             kAudioUnitScope_Global, 0, &unit_s,
+                             &(UInt32) { sizeof(unit_s) }) == noErr)
+    {
+        p_sys->au_latency_ticks = vlc_tick_from_sec(unit_s);
+        msg_Dbg(p_aout, "AudioUnit latency: %" PRId64 "us", p_sys->au_latency_ticks);
+    }
 
     return VLC_SUCCESS;
 }
diff --git a/modules/audio_output/coreaudio_common.h b/modules/audio_output/coreaudio_common.h
index 81f1a1f6d78f..e5caaa80e093 100644
--- a/modules/audio_output/coreaudio_common.h
+++ b/modules/audio_output/coreaudio_common.h
@@ -51,8 +51,6 @@ struct aout_sys_common
 {
     /* The following is owned by common.c (initialized from ca_Open) */
 
-    AudioUnit au; /* Can be NULL (pass-through) */
-
     mach_timebase_info_data_t tinfo;
 
     size_t              i_underrun_size;
@@ -73,7 +71,7 @@ struct aout_sys_common
     size_t timing_report_last_written_bytes;
     /* Number of bytes to write before sending a timing report */
     size_t timing_report_delay_bytes;
-    /* Last AudioUnit Latency, for debug/log purpose */
+    /* AudioUnit Latency */
     vlc_tick_t au_latency_ticks;
 
     union lock