Skip to content

Commit

Permalink
Provide support for JSR-310 Year and Month in IntervalShardingAlgorit…
Browse files Browse the repository at this point in the history 
…hm (apache#19139)

* Provide support for JSR-310 Year and Month in IntervalShardingAlgorithm

* fix checkstyle.

* update doc for Interval Sharding Algorithm.

* Provide support for java.time.YearMonth in IntervalShardingAlgorithm.

* fix checkstyle again.
  • Loading branch information
@linghengqian
linghengqian authored Jul 14, 2022
1 parent be6da0c commit 58dced0
Show file tree
Hide file tree
Showing 4 changed files with 216 additions and 13 deletions.
2 changes: 2 additions & 0 deletions ...cument/content/user-manual/shardingsphere-jdbc/builtin-algorithm/sharding.cn.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ Apache ShardingSphere 内置的标准分片算法实现类包括：

#### 时间范围分片算法

当传入的分片键为 `java.time.Instant` 时存在特例处理，其会携带上系统的时区信息后转化为 `datetime-pattern` 的字符串格式, 再进行下一步分片。

类型：INTERVAL

可配置属性：
Expand Down
2 changes: 2 additions & 0 deletions ...cument/content/user-manual/shardingsphere-jdbc/builtin-algorithm/sharding.en.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ Attributes:

#### Interval Sharding Algorithm

When the incoming sharding key is `java.time.Instant`, there is a special case, which will carry the time zone information of the system and convert it into the string format of `datetime-pattern`, and then proceed to the next sharding.

Type: INTERVAL

Attributes:
Expand Down
132 changes: 119 additions & 13 deletions ...apache/shardingsphere/sharding/algorithm/sharding/datetime/IntervalShardingAlgorithm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,17 @@
import org.apache.shardingsphere.sharding.api.sharding.standard.RangeShardingValue;
import org.apache.shardingsphere.sharding.api.sharding.standard.StandardShardingAlgorithm;

import java.time.ZoneId;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Month;
import java.time.Year;
import java.time.YearMonth;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
Expand Down Expand Up @@ -138,12 +142,21 @@ public Collection<String> doSharding(final Collection<String> availableTargetNam

private Collection<String> doSharding(final Collection<String> availableTargetNames, final Range<Comparable<?>> range) {
TemporalAccessor calculateTime = dateTimeLower;
LocalDate queryToLocalDate = calculateTime.query(TemporalQueries.localDate());
LocalTime queryToLocalTime = calculateTime.query(TemporalQueries.localTime());
if (null == queryToLocalTime) {
return doShardingInLocalDate(availableTargetNames, range, calculateTime);
if (!calculateTime.isSupported(ChronoField.NANO_OF_DAY)) {
if (calculateTime.isSupported(ChronoField.EPOCH_DAY)) {
return doShardingInLocalDate(availableTargetNames, range, calculateTime);
}
if (calculateTime.isSupported(ChronoField.YEAR) && calculateTime.isSupported(ChronoField.MONTH_OF_YEAR)) {
return doShardingInYearMonth(availableTargetNames, range, calculateTime);
}
if (calculateTime.isSupported(ChronoField.YEAR)) {
return doShardingInYear(availableTargetNames, range, calculateTime);
}
if (calculateTime.isSupported(ChronoField.MONTH_OF_YEAR)) {
return doShardingInMonth(availableTargetNames, range, calculateTime);
}
}
if (null == queryToLocalDate) {
if (!calculateTime.isSupported(ChronoField.EPOCH_DAY)) {
return doShardingInLocalTime(availableTargetNames, range, calculateTime);
}
return doShardingInLocalDateTime(availableTargetNames, range, calculateTime);
Expand Down Expand Up @@ -190,6 +203,48 @@ private Collection<String> doShardingInLocalDate(final Collection<String> availa
}
return result;
}

private Collection<String> doShardingInYear(final Collection<String> availableTargetNames, final Range<Comparable<?>> range, final TemporalAccessor calculateTime) {
Set<String> result = new HashSet<>();
Year dateTimeUpperAsYear = dateTimeUpper.query(Year::from);
Year dateTimeLowerAsYear = dateTimeLower.query(Year::from);
Year calculateTimeAsView = calculateTime.query(Year::from);
while (!calculateTimeAsView.isAfter(dateTimeUpperAsYear)) {
if (hasIntersection(Range.closedOpen(calculateTimeAsView, calculateTimeAsView.plus(stepAmount, stepUnit)), range, dateTimeLowerAsYear, dateTimeUpperAsYear)) {
result.addAll(getMatchedTables(calculateTimeAsView, availableTargetNames));
}
calculateTimeAsView = calculateTimeAsView.plus(stepAmount, stepUnit);
}
return result;
}

private Collection<String> doShardingInMonth(final Collection<String> availableTargetNames, final Range<Comparable<?>> range, final TemporalAccessor calculateTime) {
Set<String> result = new HashSet<>();
Month dateTimeUpperAsMonth = dateTimeUpper.query(Month::from);
Month dateTimeLowerAsMonth = dateTimeLower.query(Month::from);
Month calculateTimeAsView = calculateTime.query(Month::from);
while (!(calculateTimeAsView.getValue() > dateTimeUpperAsMonth.getValue()) && (calculateTimeAsView.getValue() + stepAmount) <= Month.DECEMBER.getValue()) {
if (hasIntersection(Range.closedOpen(calculateTimeAsView, calculateTimeAsView.plus(stepAmount)), range, dateTimeLowerAsMonth, dateTimeUpperAsMonth)) {
result.addAll(getMatchedTables(calculateTimeAsView, availableTargetNames));
}
calculateTimeAsView = calculateTimeAsView.plus(stepAmount);
}
return result;
}

private Collection<String> doShardingInYearMonth(final Collection<String> availableTargetNames, final Range<Comparable<?>> range, final TemporalAccessor calculateTime) {
Set<String> result = new HashSet<>();
YearMonth dateTimeUpperAsYearMonth = dateTimeUpper.query(YearMonth::from);
YearMonth dateTimeLowerAsYearMonth = dateTimeLower.query(YearMonth::from);
YearMonth calculateTimeAsView = calculateTime.query(YearMonth::from);
while (!calculateTimeAsView.isAfter(dateTimeUpperAsYearMonth)) {
if (hasIntersection(Range.closedOpen(calculateTimeAsView, calculateTimeAsView.plus(stepAmount, stepUnit)), range, dateTimeLowerAsYearMonth, dateTimeUpperAsYearMonth)) {
result.addAll(getMatchedTables(calculateTimeAsView, availableTargetNames));
}
calculateTimeAsView = calculateTimeAsView.plus(stepAmount, stepUnit);
}
return result;
}

private boolean hasIntersection(final Range<LocalDateTime> calculateRange, final Range<Comparable<?>> range, final LocalDateTime dateTimeLower, final LocalDateTime dateTimeUpper) {
LocalDateTime lower = range.hasLowerBound() ? parseLocalDateTime(range.lowerEndpoint()) : dateTimeLower;
Expand Down Expand Up @@ -217,6 +272,33 @@ private boolean hasIntersection(final Range<LocalTime> calculateRange, final Ran
Range<LocalTime> dateTimeRange = Range.range(lower, lowerBoundType, upper, upperBoundType);
return calculateRange.isConnected(dateTimeRange) && !calculateRange.intersection(dateTimeRange).isEmpty();
}

private boolean hasIntersection(final Range<Year> calculateRange, final Range<Comparable<?>> range, final Year dateTimeLower, final Year dateTimeUpper) {
Year lower = range.hasLowerBound() ? parseYear(range.lowerEndpoint()) : dateTimeLower;
Year upper = range.hasUpperBound() ? parseYear(range.upperEndpoint()) : dateTimeUpper;
BoundType lowerBoundType = range.hasLowerBound() ? range.lowerBoundType() : BoundType.CLOSED;
BoundType upperBoundType = range.hasUpperBound() ? range.upperBoundType() : BoundType.CLOSED;
Range<Year> dateTimeRange = Range.range(lower, lowerBoundType, upper, upperBoundType);
return calculateRange.isConnected(dateTimeRange) && !calculateRange.intersection(dateTimeRange).isEmpty();
}

private boolean hasIntersection(final Range<Month> calculateRange, final Range<Comparable<?>> range, final Month dateTimeLower, final Month dateTimeUpper) {
Month lower = range.hasLowerBound() ? parseMonth(range.lowerEndpoint()) : dateTimeLower;
Month upper = range.hasUpperBound() ? parseMonth(range.upperEndpoint()) : dateTimeUpper;
BoundType lowerBoundType = range.hasLowerBound() ? range.lowerBoundType() : BoundType.CLOSED;
BoundType upperBoundType = range.hasUpperBound() ? range.upperBoundType() : BoundType.CLOSED;
Range<Month> dateTimeRange = Range.range(lower, lowerBoundType, upper, upperBoundType);
return calculateRange.isConnected(dateTimeRange) && !calculateRange.intersection(dateTimeRange).isEmpty();
}

private boolean hasIntersection(final Range<YearMonth> calculateRange, final Range<Comparable<?>> range, final YearMonth dateTimeLower, final YearMonth dateTimeUpper) {
YearMonth lower = range.hasLowerBound() ? parseYearMonth(range.lowerEndpoint()) : dateTimeLower;
YearMonth upper = range.hasUpperBound() ? parseYearMonth(range.upperEndpoint()) : dateTimeUpper;
BoundType lowerBoundType = range.hasLowerBound() ? range.lowerBoundType() : BoundType.CLOSED;
BoundType upperBoundType = range.hasUpperBound() ? range.upperBoundType() : BoundType.CLOSED;
Range<YearMonth> dateTimeRange = Range.range(lower, lowerBoundType, upper, upperBoundType);
return calculateRange.isConnected(dateTimeRange) && !calculateRange.intersection(dateTimeRange).isEmpty();
}

private LocalDateTime parseLocalDateTime(final Comparable<?> endpoint) {
return LocalDateTime.parse(getDateTimeText(endpoint).substring(0, dateTimePatternLength), dateTimeFormatter);
Expand All @@ -229,6 +311,18 @@ private LocalDate parseLocalDate(final Comparable<?> endpoint) {
private LocalTime parseLocalTime(final Comparable<?> endpoint) {
return LocalTime.parse(getDateTimeText(endpoint).substring(0, dateTimePatternLength), dateTimeFormatter);
}

private Year parseYear(final Comparable<?> endpoint) {
return Year.parse(getDateTimeText(endpoint).substring(0, dateTimePatternLength), dateTimeFormatter);
}

private Month parseMonth(final Comparable<?> endpoint) {
return (Month) endpoint;
}

private YearMonth parseYearMonth(final Comparable<?> endpoint) {
return YearMonth.parse(getDateTimeText(endpoint).substring(0, dateTimePatternLength), dateTimeFormatter);
}

private String getDateTimeText(final Comparable<?> endpoint) {
if (endpoint instanceof Instant) {
Expand All @@ -244,15 +338,27 @@ private String getDateTimeText(final Comparable<?> endpoint) {
}

private Collection<String> getMatchedTables(final TemporalAccessor dateTime, final Collection<String> availableTargetNames) {
LocalDate localDate = dateTime.query(TemporalQueries.localDate());
LocalTime localTime = dateTime.query(TemporalQueries.localTime());
String tableSuffix;
if (null == localTime) {
tableSuffix = localDate.format(tableSuffixPattern);
return availableTargetNames.parallelStream().filter(each -> each.endsWith(tableSuffix)).collect(Collectors.toSet());
if (!dateTime.isSupported(ChronoField.NANO_OF_DAY)) {
if (dateTime.isSupported(ChronoField.EPOCH_DAY)) {
tableSuffix = tableSuffixPattern.format(dateTime.query(TemporalQueries.localDate()));
return availableTargetNames.parallelStream().filter(each -> each.endsWith(tableSuffix)).collect(Collectors.toSet());
}
if (dateTime.isSupported(ChronoField.YEAR) && dateTime.isSupported(ChronoField.MONTH_OF_YEAR)) {
tableSuffix = tableSuffixPattern.format(dateTime.query(YearMonth::from));
return availableTargetNames.parallelStream().filter(each -> each.endsWith(tableSuffix)).collect(Collectors.toSet());
}
if (dateTime.isSupported(ChronoField.YEAR)) {
tableSuffix = tableSuffixPattern.format(dateTime.query(Year::from));
return availableTargetNames.parallelStream().filter(each -> each.endsWith(tableSuffix)).collect(Collectors.toSet());
}
if (dateTime.isSupported(ChronoField.MONTH_OF_YEAR)) {
tableSuffix = tableSuffixPattern.format(dateTime.query(Month::from));
return availableTargetNames.parallelStream().filter(each -> each.endsWith(tableSuffix)).collect(Collectors.toSet());
}
}
if (null == localDate) {
tableSuffix = localTime.format(tableSuffixPattern);
if (!dateTime.isSupported(ChronoField.EPOCH_DAY)) {
tableSuffix = dateTime.query(TemporalQueries.localTime()).format(tableSuffixPattern);
return availableTargetNames.parallelStream().filter(each -> each.endsWith(tableSuffix)).collect(Collectors.toSet());
}
tableSuffix = LocalDateTime.from(dateTime).format(tableSuffixPattern);
Expand Down
93 changes: 93 additions & 0 deletions ...he/shardingsphere/sharding/algorithm/sharding/datetime/IntervalShardingAlgorithmTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,11 @@
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Month;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
import java.time.Year;
import java.time.YearMonth;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
Expand All @@ -60,6 +63,12 @@ public final class IntervalShardingAlgorithmTest {

private final Collection<String> availableTablesForJDBCTimeDataSources = new LinkedList<>();

private final Collection<String> availableTablesForYearDataSources = new LinkedList<>();

private final Collection<String> availableTablesForYearMonthDataSources = new LinkedList<>();

private final Collection<String> availableTablesForMonthInJSR310DataSources = new LinkedList<>();

private final Collection<String> availableTablesForDayWithMillisecondDataSources = new LinkedList<>();

private IntervalShardingAlgorithm shardingAlgorithmByQuarter;
Expand All @@ -74,6 +83,12 @@ public final class IntervalShardingAlgorithmTest {

private IntervalShardingAlgorithm shardingAlgorithmByDayWithMillisecond;

private IntervalShardingAlgorithm shardingAlgorithmByYear;

private IntervalShardingAlgorithm shardingAlgorithmByYearMonth;

private IntervalShardingAlgorithm shardingAlgorithmByMonthInJSR310;

@Before
public void setup() {
initShardStrategyByMonth();
Expand All @@ -82,6 +97,9 @@ public void setup() {
initShardStrategyByDayWithMillisecond();
initShardingStrategyByJDBCDate();
initShardingStrategyByJDBCTime();
initShardingStrategyByYear();
initShardingStrategyByYearMonth();
initShardingStrategyByMonthInJSR310();
}

private void initShardStrategyByQuarter() {
Expand Down Expand Up @@ -207,6 +225,68 @@ private Properties createJDBCTimeProperties(final int stepAmount) {
result.setProperty("datetime-interval-unit", "Hours");
return result;
}

private void initShardingStrategyByYear() {
int stepAmount = 2;
shardingAlgorithmByYear = (IntervalShardingAlgorithm) ShardingAlgorithmFactory.newInstance(
new ShardingSphereAlgorithmConfiguration("INTERVAL", createYearProperties(stepAmount)));
for (int i = 2000; i < 2023; i++) {
availableTablesForYearDataSources.add(String.format("t_order_%04d", i));
}
}

private Properties createYearProperties(final int stepAmount) {
Properties result = new Properties();
result.setProperty("datetime-pattern", "yyyy");
result.setProperty("datetime-lower", "2000");
result.setProperty("datetime-upper", "2022");
result.setProperty("sharding-suffix-pattern", "yyyy");
result.setProperty("datetime-interval-amount", Integer.toString(stepAmount));
result.setProperty("datetime-interval-unit", "Years");
return result;
}

private void initShardingStrategyByYearMonth() {
int stepAmount = 2;
shardingAlgorithmByYearMonth = (IntervalShardingAlgorithm) ShardingAlgorithmFactory.newInstance(
new ShardingSphereAlgorithmConfiguration("INTERVAL", createYearMonthProperties(stepAmount)));
for (int i = 2016; i <= 2021; i++) {
for (int j = 1; j <= 12; j++) {
availableTablesForYearMonthDataSources.add(String.format("t_order_%04d%02d", i, j));
}
}
}

private Properties createYearMonthProperties(final int stepAmount) {
Properties result = new Properties();
result.setProperty("datetime-pattern", "yyyy-MM");
result.setProperty("datetime-lower", "2016-01");
result.setProperty("datetime-upper", "2021-12");
result.setProperty("sharding-suffix-pattern", "yyyyMM");
result.setProperty("datetime-interval-amount", Integer.toString(stepAmount));
result.setProperty("datetime-interval-unit", "Years");
return result;
}

private void initShardingStrategyByMonthInJSR310() {
int stepAmount = 2;
shardingAlgorithmByMonthInJSR310 = (IntervalShardingAlgorithm) ShardingAlgorithmFactory.newInstance(
new ShardingSphereAlgorithmConfiguration("INTERVAL", createMonthInJSR310Properties(stepAmount)));
for (int i = 2; i < 13; i++) {
availableTablesForMonthInJSR310DataSources.add(String.format("t_order_%02d", i));
}
}

private Properties createMonthInJSR310Properties(final int stepAmount) {
Properties result = new Properties();
result.setProperty("datetime-pattern", "MM");
result.setProperty("datetime-lower", "02");
result.setProperty("datetime-upper", "12");
result.setProperty("sharding-suffix-pattern", "MM");
result.setProperty("datetime-interval-amount", Integer.toString(stepAmount));
result.setProperty("datetime-interval-unit", "Months");
return result;
}

@Test
public void assertPreciseDoShardingByQuarter() {
Expand Down Expand Up @@ -352,4 +432,17 @@ public void assertTimeInJDBCType() {
OffsetTime.of(12, 25, 27, 0, OffsetDateTime.now().getOffset()))));
assertThat(actualAsOffsetTime.size(), is(6));
}

@Test
public void assertIntegerInJDBCType() {
Collection<String> actualAsYear = shardingAlgorithmByYear.doSharding(availableTablesForYearDataSources,
new RangeShardingValue<>("t_order", "create_time", DATA_NODE_INFO, Range.closed(Year.of(2001), Year.of(2013))));
assertThat(actualAsYear.size(), is(7));
Collection<String> actualAsYearMonth = shardingAlgorithmByYearMonth.doSharding(availableTablesForYearMonthDataSources,
new RangeShardingValue<>("t_order", "create_time", DATA_NODE_INFO, Range.closed(YearMonth.of(2016, 1), YearMonth.of(2020, 1))));
assertThat(actualAsYearMonth.size(), is(3));
Collection<String> actualAsMonth = shardingAlgorithmByMonthInJSR310.doSharding(availableTablesForMonthInJSR310DataSources,
new RangeShardingValue<>("t_order", "create_time", DATA_NODE_INFO, Range.closed(Month.of(4), Month.of(10))));
assertThat(actualAsMonth.size(), is(4));
}
}

0 comments on commit 58dced0

Please sign in to comment.