Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactoring Storage #12119

Open
wants to merge 37 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
0c34336
Update Interface.
jdunkerley Jan 22, 2025
e6d714c
Update base Storage and SpecializedStorage.
jdunkerley Jan 22, 2025
c95a526
Update BigDecimalStorage.
jdunkerley Jan 22, 2025
e635538
Update BigIntegerStorage.
jdunkerley Jan 22, 2025
4991572
Fix as ColumnStorage is now generic.
jdunkerley Jan 22, 2025
50328f3
Update the datetime storages.
jdunkerley Jan 22, 2025
8add472
Update StringStorage.
jdunkerley Jan 22, 2025
e3e69f6
Update ObjectStorage and MixedStorage.
jdunkerley Jan 22, 2025
8bf5b45
Removing StringStorage constructors in favour of builders.
jdunkerley Jan 22, 2025
56989a0
Work on Long and Double storage.
jdunkerley Jan 22, 2025
49f36ae
Remove a fair amount of `new LongStorage`.
jdunkerley Jan 23, 2025
1f18276
Java formatting.
jdunkerley Jan 23, 2025
6158438
Java formatting.
jdunkerley Jan 23, 2025
e0c6d6c
More WIP.
jdunkerley Jan 23, 2025
156fd34
Rename `get` to `getPrimitive` so can find easily.
jdunkerley Jan 24, 2025
2aa0865
Working through `size`.
jdunkerley Jan 24, 2025
8e3a2fc
Working through `size` (2).
jdunkerley Jan 24, 2025
dbe4b3a
Working through `size` (3).
jdunkerley Jan 24, 2025
1572a4e
Fix build issue.
jdunkerley Jan 24, 2025
a0c90cc
Some tests fixed.
jdunkerley Jan 24, 2025
598efaa
Use long indexes.
jdunkerley Jan 24, 2025
b265313
Most of size gone and most of constructor use replaces.
jdunkerley Jan 24, 2025
8ad8adc
Basically complete.
jdunkerley Jan 24, 2025
f95c03a
Rebase fixes.
jdunkerley Jan 25, 2025
d8783ef
Java formatting.
jdunkerley Jan 26, 2025
1af40f2
Revert getBoxed to getItemBoxed.
jdunkerley Jan 26, 2025
27bcd77
Revert getPrimitive renames.
jdunkerley Jan 26, 2025
5d69693
Java format.
jdunkerley Jan 26, 2025
415c37b
PR comments.
jdunkerley Jan 27, 2025
65231cd
Change order of arguments.
jdunkerley Jan 27, 2025
40e3527
Rename getItemDouble back to getItemAsDouble.
jdunkerley Jan 27, 2025
199e997
Rename getItemLong and getItemBoolean to
jdunkerley Jan 27, 2025
45f3827
Self PR fixes to Builders.
jdunkerley Jan 27, 2025
d2a98ce
Self PR fixes to Cast.
jdunkerley Jan 27, 2025
2917ea1
Use same integer type when slicing, masking, filtering.
jdunkerley Jan 27, 2025
ecd0934
Update BigIntegerArrayAdapter to long.
jdunkerley Jan 27, 2025
738689a
Update DoubleArrayAdapter to long.
jdunkerley Jan 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
More WIP.
  • Loading branch information
jdunkerley committed Jan 25, 2025
commit e0c6d6c2068c9cd2aa52a0795a4e88575c04dfca
8 changes: 4 additions & 4 deletions distribution/lib/Standard/Table/0.0.0-dev/src/Column.enso
Original file line number Diff line number Diff line change
Expand Up @@ -1507,7 +1507,7 @@ type Column

builder = make_string_builder length
0.up_to length . each i->
replaced = do_replace i (storage.getItemBoxed i)
replaced = do_replace i (storage.getBoxed i)
builder.append replaced

Column.from_storage new_name builder.seal
Expand Down Expand Up @@ -2294,7 +2294,7 @@ type Column
valid_index = (index >= 0) && (index < self.length)
if valid_index.not then default else
storage = self.java_column.getStorage
java_to_enso <| storage.getItemBoxed index
java_to_enso <| storage.getBoxed index

## PRIVATE
ICON data_input
Expand Down Expand Up @@ -2363,7 +2363,7 @@ type Column
to_js_object self =
name = self.java_column.getName
storage = self.java_column.getStorage
storage_proxy = Array_Proxy.new storage.size i-> storage.getItemBoxed i
storage_proxy = Array_Proxy.new storage.size i-> storage.getBoxed i
storage_json = Vector.from_polyglot_array storage_proxy
JS_Object.from_pairs [["name", name], ["data", storage_json]]

Expand Down Expand Up @@ -2859,7 +2859,7 @@ wrap_text_argument_as_value_provider val =
text : Text -> (_-> text)
col : Column ->
storage = col.java_column.getStorage
i-> storage.getItemBoxed i
i-> storage.getBoxed i

## PRIVATE
wrap_text_or_regex_argument_as_value_provider val =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ map_over_storage input_column function builder skip_nothing=True on_problems:Pro
num_input_rows = input_storage.size
output_storage_builder = builder num_input_rows
0.up_to num_input_rows . each i->
input_value = input_storage.getItemBoxed i
input_value = input_storage.getBoxed i
if skip_nothing && input_value.is_nothing then output_storage_builder.append Nothing else
output_value = function input_value . catch Any err->
problem_builder.report_other_warning err
Expand All @@ -41,8 +41,8 @@ map_2_over_storage input_column_0 input_column_1 function builder skip_nothing=T
num_input_rows = input_storage_0.size
output_storage_builder = builder num_input_rows
ok = 0.up_to num_input_rows . each_propagate i->
input_value_0 = input_storage_0.getItemBoxed i
input_value_1 = input_storage_1.getItemBoxed i
input_value_0 = input_storage_0.getBoxed i
input_value_1 = input_storage_1.getBoxed i
if skip_nothing && input_value_0.is_nothing then output_storage_builder.append Nothing else
output_value = function input_value_0 input_value_1
output_storage_builder.append output_value
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ ansi_bold enabled txt =
## PRIVATE
Gets a textual representation of the item at position `ix` in `storage`.
private _get_item_as_text storage ix =
item = storage.getItemBoxed ix
item = storage.getBoxed ix
## Special handling for display of Text to avoid quotes when not necessary.
case item of
_ : Text -> _normalize_text_for_display item
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ fan_out_to_rows_and_columns_fixed input_storage function at_least_one_row:Boolea
maybe_add_empty_row vecs = if vecs.is_empty && at_least_one_row then empty_row else vecs

0.up_to num_input_rows . each i->
input_value = input_storage.getItemBoxed i
input_value = input_storage.getBoxed i
output_values = maybe_add_empty_row (function input_value)

output_values.each row_unchecked->
Expand Down Expand Up @@ -192,7 +192,7 @@ fan_out_to_rows_and_columns_dynamic input_storage function at_least_one_row colu
[Vector.fill output_column_builders.length Nothing]

0.up_to num_input_rows . each i->
input_value = input_storage.getItemBoxed i
input_value = input_storage.getBoxed i
output_values = maybe_add_empty_row (function input_value)

# get the column names for the row.
Expand Down Expand Up @@ -250,7 +250,7 @@ map_columns_to_multiple input_column function column_count column_builder=make_s
builders = Builder.new

0.up_to num_rows . each i->
input_value = input_storage.getItemBoxed i
input_value = input_storage.getBoxed i
output_values = function input_value

# Add more builders if necessary to accommodate `output_values`.
Expand All @@ -276,7 +276,7 @@ map_columns_to_multiple input_column function column_count column_builder=make_s
builders = Vector.new column_count (_-> column_builder num_rows)

output_lengths = 0.up_to num_rows . map i->
input_value = input_storage.getItemBoxed i
input_value = input_storage.getBoxed i
output_values = function input_value

## Add `output_values` to builders; if there are more builders
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
Context context = Context.getCurrent();
StringBuilder current = null;
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value == null || value instanceof String) {
String textValue = toQuotedString(value, quote, separator);

if (!separator.equals("") && quote.equals("") && textValue.contains(separator)) {
if (!separator.isEmpty() && quote.isEmpty() && textValue.contains(separator)) {
innerAggregator.reportColumnAggregatedProblem(
new UnquotedDelimiter(this.getName(), row, "Unquoted delimiter."));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
Context context = Context.getCurrent();
int count = 0;
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value == null) {
count += isEmpty ? 1 : 0;
} else if (value instanceof String asString) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
Context context = Context.getCurrent();
long count = 0;
for (int row : indexes) {
count += ((storage.getItemBoxed(row) == null) == isNothing ? 1L : 0L);
count += ((storage.getBoxed(row) == null) == isNothing ? 1L : 0L);
context.safepoint();
}
return count;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ private Object firstBySpecifiedOrder(List<Integer> indexes) {

Context context = Context.getCurrent();
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (ignoreNothing && value == null) {
continue;
}
Expand All @@ -62,7 +62,7 @@ private Object firstBySpecifiedOrder(List<Integer> indexes) {
new OrderedMultiValueKey(this.orderByColumns, row, this.orderByDirections);
if (key == null || key.compareTo(newKey) > 0) {
key = newKey;
current = storage.getItemBoxed(row);
current = storage.getBoxed(row);
}

context.safepoint();
Expand All @@ -74,7 +74,7 @@ private Object firstBySpecifiedOrder(List<Integer> indexes) {
private Object firstByRowOrder(List<Integer> indexes) {
Context context = Context.getCurrent();
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (!ignoreNothing || value != null) {
return value;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ public GroupBy(String name, Column column) {

@Override
public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregator) {
return indexes.isEmpty() ? null : storage.getItemBoxed(indexes.get(0));
return indexes.isEmpty() ? null : storage.getBoxed(indexes.getFirst());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ private Object lastBySpecifiedOrder(List<Integer> indexes) {
Context context = Context.getCurrent();
for (int i = indexes.size() - 1; i >= 0; i--) {
int row = indexes.get(i);
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (ignoreNothing && value == null) {
continue;
}
Expand All @@ -62,7 +62,7 @@ private Object lastBySpecifiedOrder(List<Integer> indexes) {
new OrderedMultiValueKey(this.orderByColumns, row, this.orderByDirections);
if (key == null || key.compareTo(newKey) < 0) {
key = newKey;
current = storage.getItemBoxed(row);
current = storage.getBoxed(row);
}

context.safepoint();
Expand All @@ -74,7 +74,7 @@ private Object lastBySpecifiedOrder(List<Integer> indexes) {
private Object lastByRowOrder(List<Integer> indexes) {
Context context = Context.getCurrent();
for (int i = indexes.size() - 1; i >= 0; i--) {
Object value = storage.getItemBoxed(indexes.get(i));
Object value = storage.getBoxed(indexes.get(i));
if (!ignoreNothing || value != null) {
return value;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ void accumulate(
ColumnAggregatedProblemAggregator innerAggregator =
new ColumnAggregatedProblemAggregator(problemAggregator);
for (int i : indexes) {
Object value = storage.getItemBoxed(i);
Object value = storage.getBoxed(i);
if (value != null) {
Double dValue = NumericConverter.tryConvertingToDouble(value);
if (dValue == null) {
Expand Down Expand Up @@ -136,7 +136,7 @@ void accumulate(
new ColumnAggregatedProblemAggregator(problemAggregator);
Context context = Context.getCurrent();
for (int i : indexes) {
Object value = storage.getItemBoxed(i);
Object value = storage.getBoxed(i);
if (value != null) {
try {
BigDecimal valueAsBigDecimal = NumericConverter.coerceToBigDecimal(value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
Context context = Context.getCurrent();
Object current = null;
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value != null) {
try {
if (current == null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
int count = 0;
Map<Object, Integer> currentMap = null;
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value != null) {
// Merge all numbers onto a Long if possible or a Double if needed
Long lValue = NumericConverter.tryConvertingToLong(value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
int count = 0;
SortedMap<Double, Integer> currentMap = new TreeMap<>();
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value != null) {
Double dValue = NumericConverter.tryConvertingToDouble(value);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
Object current = null;

for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value != null) {
if (!(value instanceof String asString)) {
innerAggregator.reportColumnAggregatedProblem(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public Object aggregate(List<Integer> indexes, ProblemAggregator problemAggregat
Context context = Context.getCurrent();
Calculation current = null;
for (int row : indexes) {
Object value = storage.getItemBoxed(row);
Object value = storage.getBoxed(row);
if (value != null) {
Double dValue = NumericConverter.tryConvertingToDouble(value);
if (dValue == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
import org.enso.table.data.column.builder.Builder;
import org.enso.table.data.column.builder.InferredIntegerBuilder;
import org.enso.table.data.column.operation.map.MapOperationProblemAggregator;
import org.enso.table.data.column.storage.ColumnDoubleStorage;
import org.enso.table.data.column.storage.ColumnLongStorage;
import org.enso.table.data.column.storage.Storage;
import org.enso.table.data.column.storage.numeric.BigIntegerStorage;
import org.enso.table.data.column.storage.numeric.DoubleStorage;
import org.enso.table.data.column.storage.type.BigIntegerType;
import org.enso.table.data.column.storage.type.FloatType;
import org.enso.table.data.column.storage.type.IntegerType;
Expand Down Expand Up @@ -169,7 +169,7 @@ void add(Object value) {
@Override
void accumulate(List<Integer> indexes, Storage<?> storage) {
Context context = Context.getCurrent();
if (storage instanceof DoubleStorage doubleStorage) {
if (storage instanceof ColumnDoubleStorage doubleStorage) {
for (int row : indexes) {
if (!doubleStorage.isNothing(row)) {
addDouble(doubleStorage.get(row));
Expand All @@ -178,7 +178,7 @@ void accumulate(List<Integer> indexes, Storage<?> storage) {
}
} else {
for (int row : indexes) {
add(storage.getItemBoxed(row));
add(storage.getBoxed(row));
context.safepoint();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public void appendBulkStorage(Storage<?> storage) {
if (storage instanceof DateStorage dateStorage) {
Context context = Context.getCurrent();
for (int i = 0; i < dateStorage.size(); ++i) {
LocalDate date = dateStorage.getItemBoxed(i);
LocalDate date = dateStorage.getBoxed(i);
if (date == null) {
data[currentSize++] = null;
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public void appendBulkStorage(Storage<?> storage) {
appendNulls(storage.size());
} else {
for (int i = 0; i < storage.size(); i++) {
append(storage.getItemBoxed(i));
append(storage.getBoxed(i));
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public void appendBulkStorage(Storage<?> storage) {
appendNulls(storage.size());
} else {
for (int i = 0; i < storage.size(); i++) {
append(storage.getItemBoxed(i));
append(storage.getBoxed(i));
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public void appendBulkStorage(Storage<?> storage) {
} else {
int n = storage.size();
for (int i = 0; i < n; i++) {
data[currentSize++] = storage.getItemBoxed(i);
data[currentSize++] = storage.getBoxed(i);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ private Storage<BigInteger> convertBigDecimalStorage(
Builder.getForBigInteger(bigDecimalStorage.size(), problemAggregator),
bigDecimalStorage,
(i) -> {
BigDecimal x = bigDecimalStorage.getBoxed((int) i);
BigDecimal x = bigDecimalStorage.getBoxed(i);
return x.toBigInteger();
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,14 @@ private Storage<Double> convertBigIntegerStorage(
return innerLoop(
Builder.getForDouble(FloatType.FLOAT_64, storage.size(), problemAggregator),
storage,
(builder, i) -> builder.append(storage.getItemBoxed((int) i)));
(builder, i) -> builder.append(storage.getBoxed(i)));
}

private Storage<Double> convertBigDecimalStorage(
Storage<BigDecimal> storage, CastProblemAggregator problemAggregator) {
return innerLoop(
Builder.getForDouble(FloatType.FLOAT_64, storage.size(), problemAggregator),
storage,
(builder, i) -> builder.append(storage.getItemBoxed((int) i)));
(builder, i) -> builder.append(storage.getBoxed(i)));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ private Storage<Long> convertBigIntegerStorage(
Builder.getForLong(targetType, storage.size(), problemAggregator),
storage,
(builder, i) -> {
BigInteger value = storage.getBoxed((int) i);
BigInteger value = storage.getBoxed(i);
if (targetType.fits(value)) {
builder.appendLong(value.longValue());
} else {
Expand All @@ -182,7 +182,7 @@ private Storage<Long> convertBigDecimalStorage(
Builder.getForLong(targetType, storage.size(), problemAggregator),
storage,
(builder, i) -> {
BigDecimal value = storage.getItemBoxed((int) i);
BigDecimal value = storage.getBoxed(i);
BigInteger bigInteger = value.toBigInteger();
if (targetType.fits(bigInteger)) {
builder.appendLong(bigInteger.longValue());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private <T> Storage<String> castTemporalStorage(
Builder.getForText(storage.size(), targetType),
storage,
(i) -> {
var value = storage.getItemBoxed((int) i);
var value = storage.getBoxed(i);
return adapt(converter.apply(value), problemAggregator);
});
}
Expand Down
Loading