Skip to content

Commit

Permalink
Standardize use of transmit_frequency_start/stop and use `pulse_for…
Browse files Browse the repository at this point in the history 
…m` for BB checks (OSOceanAcoustics#1091)

* docs: remove outdated Attention box in Data Processing page

* In EK80, rename frequency_start/end to convention-based transmit_frequency_start/stop

* For EK80 transmit_frequency_start/stop, standardize data type (int to float) and attributes

* Add transmit_frequency_start/stop to EK60 & AZFP, set to frequency_nominal

* Replace use of frequency_start/stop in ek80 checks for BB channels with pulse_form

* Rename pulse_form to transmit_type per convention, and map int codes to strings (CW, FM, FMD)

* Assigned attributes to the wrong variable, in previous commit

* The meaning for ek80 channel_mode = 1 is not described in ek80 manual

* Ensure variables remain encoded as int types even after xr.merge

* Overhaul EK80 checks for BB vs CW to use pulse_form / transmit_type

* Set transmit_frequency_start/stop attributes in 1.0.yml and update set_groups_azfp/ek60 to use that

* For ek80 transmit_freq_start/stop is now created and/or populated with frequency_nominal for power-only channels. Updated all BB vs CW checks.

* Simplified ek80 tapered_chirp now that CW channels always contain freq nominal in transmit_frequency_start/stop

* Small cleanups to comments, unused function parameter, etc, all involving ek80

* Simplify ek80 test for encode_mode = 'complex' but there are no CW pings

* Remove transmit_type from tx_param_names b/c it's no longer an argument in tapered_chirp

* Optimize ek80 BB vs CW tests using transmit_type on first ping_time

* Update echopype/echodata/simrad.py

Co-authored-by: Wu-Jung Lee <[email protected]>

* Update echopype/echodata/simrad.py

Co-authored-by: Wu-Jung Lee <[email protected]>

* Change transmit_type FM string to LFM, per convention, and add more detailed flag_meanings strings, largely from the convention

---------

Co-authored-by: Wu-Jung Lee <[email protected]>
  • Loading branch information
@emiliom @leewujung
emiliom and leewujung authored Aug 4, 2023
1 parent 389802a commit 6671fec
Show file tree
Hide file tree
Showing 9 changed files with 154 additions and 115 deletions.
32 changes: 19 additions & 13 deletions echopype/calibrate/calibrate_ek.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,9 @@ def __init__(
# Use center frequency if in BB mode, else use nominal channel frequency
if self.waveform_mode == "BB":
# use true center frequency to interpolate for various cal params
self.freq_center = (beam["frequency_start"] + beam["frequency_end"]).sel(
channel=self.chan_sel
) / 2
self.freq_center = (
beam["transmit_frequency_start"] + beam["transmit_frequency_stop"]
).sel(channel=self.chan_sel) / 2
else:
# use nominal channel frequency for CW pulse
self.freq_center = beam["frequency_nominal"].sel(channel=self.chan_sel)
Expand Down Expand Up @@ -309,18 +309,24 @@ def _get_chan_dict(beam: xr.Dataset) -> Dict:
"""
# Use center frequency for each ping to select BB or CW channels
# when all samples are encoded as complex samples
if "frequency_start" in beam and "frequency_end" in beam:
# At least some channels are BB
# frequency_start and frequency_end are NaN for CW channels
freq_center = (beam["frequency_start"] + beam["frequency_end"]) / 2

if not np.all(beam["transmit_type"] == "CW"):
# At least 1 BB ping exists -- this is analogous to what we had from before
# Before: when at least 1 BB ping exists, frequency_start and frequency_end will exist

# assume transmit_type identical for all pings in a channel
first_ping_transmit_type = (
beam["transmit_type"].isel(ping_time=0).drop_vars("ping_time")
) # noqa
return {
# For BB: drop channels containing CW samples (nan in freq start/end)
"BB": freq_center.dropna(dim="channel").channel,
# For CW: drop channels containing BB samples (not nan in freq start/end)
"CW": freq_center.where(np.isnan(freq_center), drop=True).channel,
# For BB: Keep only non-CW channels (LFM or FMD) based on transmit_type
"BB": first_ping_transmit_type.where(
first_ping_transmit_type != "CW", drop=True
).channel, # noqa
# For CW: Keep only CW channels based on transmit_type
"CW": first_ping_transmit_type.where(
first_ping_transmit_type == "CW", drop=True
).channel, # noqa
}

else:
# All channels are CW
return {"BB": None, "CW": beam.channel}
Expand Down
35 changes: 12 additions & 23 deletions echopype/calibrate/ek80_complex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,18 @@ def tapered_chirp(
fs,
transmit_duration_nominal,
slope,
frequency_nominal=None,
frequency_start=None,
frequency_end=None,
transmit_frequency_start,
transmit_frequency_stop,
):
"""
Create the chirp replica following implementation from Lars Anderson.
Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py # noqa
"""
if frequency_start is None and frequency_end is None: # CW waveform
frequency_start = frequency_nominal
frequency_end = frequency_nominal

tau = transmit_duration_nominal
f0 = frequency_start
f1 = frequency_end
f0 = transmit_frequency_start
f1 = transmit_frequency_stop

nsamples = int(np.floor(tau * fs))
t = np.linspace(0, nsamples - 1, num=nsamples) * 1 / fs
Expand Down Expand Up @@ -229,27 +225,20 @@ def get_transmit_signal(
# Make sure it is BB mode data
# This is already checked in calibrate_ek
# but keeping this here for use as standalone function
if waveform_mode == "BB" and (("frequency_start" not in beam) or ("frequency_end" not in beam)):
if waveform_mode == "BB" and np.all(beam["transmit_type"] == "CW"):
raise TypeError("File does not contain BB mode complex samples!")

# Generate all transmit replica
y_all = {}
y_time_all = {}
# TODO: expand to deal with the case with varying tx param across ping_time
tx_param_names = [
"transmit_duration_nominal",
"slope",
"transmit_frequency_start",
"transmit_frequency_stop",
]
for ch in beam["channel"].values:
# TODO: expand to deal with the case with varying tx param across ping_time
if waveform_mode == "BB":
tx_param_names = [
"transmit_duration_nominal",
"slope",
"frequency_start",
"frequency_end",
]
else:
tx_param_names = [
"transmit_duration_nominal",
"slope",
"frequency_nominal",
]
tx_params = {}
for p in tx_param_names:
tx_params[p] = np.unique(beam[p].sel(channel=ch))
Expand Down
12 changes: 11 additions & 1 deletion echopype/convert/set_groups_azfp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,14 +408,24 @@ def set_beam(self) -> List[xr.Dataset]:
"valid_min": 0.0,
},
),
"transmit_frequency_start": (
["channel"],
self.freq_sorted,
self._varattrs["beam_var_default"]["transmit_frequency_start"],
),
"transmit_frequency_stop": (
["channel"],
self.freq_sorted,
self._varattrs["beam_var_default"]["transmit_frequency_stop"],
),
"transmit_type": (
[],
"CW",
{
"long_name": "Type of transmitted pulse",
"flag_values": ["CW"],
"flag_meanings": [
"Continuous Wave",
"Continuous Wave – a pulse nominally of one frequency",
],
},
),
Expand Down
12 changes: 11 additions & 1 deletion echopype/convert/set_groups_ek60.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,14 +585,24 @@ def set_beam(self) -> List[xr.Dataset]:
["channel"],
beam_params["gpt_software_version"],
),
"transmit_frequency_start": (
["channel"],
self.freq,
self._varattrs["beam_var_default"]["transmit_frequency_start"],
),
"transmit_frequency_stop": (
["channel"],
self.freq,
self._varattrs["beam_var_default"]["transmit_frequency_stop"],
),
"transmit_type": (
[],
"CW",
{
"long_name": "Type of transmitted pulse",
"flag_values": ["CW"],
"flag_meanings": [
"Continuous Wave",
"Continuous Wave – a pulse nominally of one frequency",
],
},
),
Expand Down
134 changes: 77 additions & 57 deletions echopype/convert/set_groups_ek80.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,11 @@ def _assemble_ds_ping_invariant(self, params, data_type):
"standard_name": "sound_frequency",
},
),
"beam_type": (["channel"], beam_params["transducer_beam_type"]),
"beam_type": (
["channel"],
beam_params["transducer_beam_type"],
{"long_name": "type of transducer (0-single, 1-split)"},
),
"beamwidth_twoway_alongship": (
["channel"],
beam_params["beam_width_alongship"],
Expand Down Expand Up @@ -622,15 +626,29 @@ def _assemble_ds_ping_invariant(self, params, data_type):
attrs={"beam_mode": "vertical", "conversion_equation_t": "type_3"},
)

if data_type == "power":
ds = ds.assign(
{
"transmit_frequency_start": (
["channel"],
freq,
self._varattrs["beam_var_default"]["transmit_frequency_start"],
),
"transmit_frequency_stop": (
["channel"],
freq,
self._varattrs["beam_var_default"]["transmit_frequency_stop"],
),
}
)

return ds

def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset:
"""
Returns a Dataset with variables
``frequency_start`` and ``frequency_end``
added to ``ds_tmp`` for a specific channel,
if ``frequency_start`` is in ping_data_dict.
``transmit_frequency_start`` and ``transmit_frequency_stop``
added to ``ds_tmp`` for a specific channel.
Parameters
----------
ds_tmp: xr.Dataset
Expand All @@ -639,54 +657,43 @@ def _add_freq_start_end_ds(self, ds_tmp: xr.Dataset, ch: str) -> xr.Dataset:
Channel id
"""

# Process if it's a BB channel (not all pings are CW, where pulse_form encodes CW as 0)
# CW data encoded as complex samples do NOT have frequency_start and frequency_end
# TODO: use PulseForm instead of checking for the existence
# of FrequencyStart and FrequencyEnd
if (
"frequency_start" in self.parser_obj.ping_data_dict.keys()
and self.parser_obj.ping_data_dict["frequency_start"][ch]
):
ds_f_start_end = xr.Dataset(
{
"frequency_start": (
["ping_time"],
np.array(
self.parser_obj.ping_data_dict["frequency_start"][ch],
dtype=int,
),
{
"long_name": "Starting frequency of the transducer",
"units": "Hz",
},
),
"frequency_end": (
["ping_time"],
np.array(
self.parser_obj.ping_data_dict["frequency_end"][ch],
dtype=int,
),
{
"long_name": "Ending frequency of the transducer",
"units": "Hz",
},
),
},
coords={
"ping_time": (
["ping_time"],
self.parser_obj.ping_time[ch],
{
"axis": "T",
"long_name": "Timestamp of each ping",
"standard_name": "time",
},
),
},
)
if not np.all(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch]) == 0):
freq_start = np.array(self.parser_obj.ping_data_dict["frequency_start"][ch])
freq_stop = np.array(self.parser_obj.ping_data_dict["frequency_end"][ch])
elif not self.sorted_channel["power"]:
freq = self.parser_obj.config_datagram["configuration"][ch]["transducer_frequency"]
freq_start = np.full(len(self.parser_obj.ping_time[ch]), freq)
freq_stop = freq_start
else:
return ds_tmp

ds_tmp = xr.merge(
[ds_tmp, ds_f_start_end], combine_attrs="override"
) # override keeps the Dataset attributes
ds_f_start_end = xr.Dataset(
{
"transmit_frequency_start": (
["ping_time"],
freq_start.astype(float),
self._varattrs["beam_var_default"]["transmit_frequency_start"],
),
"transmit_frequency_stop": (
["ping_time"],
freq_stop.astype(float),
self._varattrs["beam_var_default"]["transmit_frequency_stop"],
),
},
coords={
"ping_time": (
["ping_time"],
self.parser_obj.ping_time[ch],
self._varattrs["beam_coord_default"]["ping_time"],
),
},
)

ds_tmp = xr.merge(
[ds_tmp, ds_f_start_end], combine_attrs="override"
) # override keeps the Dataset attributes

return ds_tmp

Expand Down Expand Up @@ -874,6 +881,8 @@ def _assemble_ds_power(self, ch):
}
)

ds_tmp = self._add_freq_start_end_ds(ds_tmp, ch)

return set_time_encodings(ds_tmp)

def _assemble_ds_common(self, ch, range_sample_size):
Expand All @@ -888,6 +897,10 @@ def _assemble_ds_common(self, ch, range_sample_size):
self.parser_obj.ping_data_dict["pulse_duration"][ch], dtype="float32"
)

def pulse_form_map(pulse_form):
str_map = np.array(["CW", "LFM", "", "", "", "FMD"])
return str_map[pulse_form]

ds_common = xr.Dataset(
{
"sample_interval": (
Expand Down Expand Up @@ -928,16 +941,23 @@ def _assemble_ds_common(self, ch, range_sample_size):
{
"long_name": "Transceiver mode",
"flag_values": [0, 1],
"flag_meanings": ["Active", "Inactive"],
"flag_meanings": ["Active", "Unknown"],
},
),
"pulse_form": (
"transmit_type": (
["ping_time"],
np.array(self.parser_obj.ping_data_dict["pulse_form"][ch], dtype=np.byte),
pulse_form_map(np.array(self.parser_obj.ping_data_dict["pulse_form"][ch])),
{
"long_name": "Pulse type",
"flag_values": [0, 1, 5],
"flag_meanings": ["CW", "FM", "FMD"],
"long_name": "Type of transmitted pulse",
"flag_values": ["CW", "LFM", "FMD"],
"flag_meanings": [
"Continuous Wave – a pulse nominally of one frequency",
"Linear Frequency Modulation – a pulse which varies from "
"transmit_frequency_start to transmit_frequency_stop in a linear "
"manner over the nominal pulse duration (transmit_duration_nominal)",
"Frequency Modulated 'D' - An EK80-specific FM type that is not "
"clearly described",
],
},
),
"sample_time_offset": (
Expand Down
10 changes: 10 additions & 0 deletions echopype/echodata/convention/1.0.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,16 @@ variable_and_varattributes:
long_name: Raw backscatter measurements (real part)
backscatter_i:
long_name: Raw backscatter measurements (imaginary part)
transmit_frequency_start:
long_name: Start frequency in transmitted pulse
units: Hz
standard_name: sound_frequency
valid_min: 0.0
transmit_frequency_stop:
long_name: Stop frequency in transmitted pulse
units: Hz
standard_name: sound_frequency
valid_min: 0.0
platform_coord_default:
time1:
axis: T
Expand Down
Loading

0 comments on commit 6671fec

Please sign in to comment.