Skip to content

Commit

Permalink
[HOPSWORKS-2918][Append] Register built-in transformations by defaut …
Browse files Browse the repository at this point in the history 
…for every project in backend (logicalclocks#812) (logicalclocks#1034)
  • Loading branch information
@moritzmeister
moritzmeister authored Feb 21, 2022
1 parent 5eecc76 commit f835c26
Show file tree
Hide file tree
Showing 12 changed files with 147 additions and 56 deletions.
2 changes: 1 addition & 1 deletion alerting/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<artifactId>mockito-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand Down
4 changes: 2 additions & 2 deletions hopsworks-alert/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.19</version>
<artifactId>mockito-core</artifactId>
<version>4.3.1</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
13 changes: 5 additions & 8 deletions hopsworks-alert/src/test/java/TestAlertManagerConfigTimer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,6 @@
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;

public class TestAlertManagerConfigTimer {
private final static Logger LOGGER = Logger.getLogger(TestAlertManagerConfigTimer.class.getName());
private AlertManagerClient client;
Expand Down Expand Up @@ -202,12 +199,12 @@ public void setUp()
Object[] args = invocation.getArguments();
LOGGER.log(Level.INFO, "Save to database: {0}.", args);
return null;
}).when(alertManagerConfigFacade).save(anyObject());
}).when(alertManagerConfigFacade).save(Mockito.any());
Mockito.doAnswer((Answer<Void>) invocation -> {
Object[] args = invocation.getArguments();
LOGGER.log(Level.INFO, "Update database: {0}.", args);
return null;
}).when(alertManagerConfigFacade).update(anyObject());
}).when(alertManagerConfigFacade).update(Mockito.any());
AlertManagerConfig alertManagerConfig = read(new File(alertManagerConfigDBPath));
AlertManagerConfigEntity alertManagerConfigEntity = new AlertManagerConfigEntity();
alertManagerConfigEntity.setId(1);
Expand All @@ -222,7 +219,7 @@ public void setUp()
Optional<AlertReceiver> alertReceiver =
receivers.stream().filter(receiver -> receiver.getName().equals(args[0])).findFirst();
return alertReceiver;
}).when(alertReceiverFacade).findByName(anyString());
}).when(alertReceiverFacade).findByName(Mockito.any());
alertManagerConfiguration = new AlertManagerConfiguration(client, alertManagerConfigController,
alertManagerConfigFacade, alertReceiverFacade);
alertManagerConfigBackup = alertManagerConfigController.read();
Expand All @@ -244,7 +241,7 @@ public void testRestoreBackup()
assert alertManagerConfig.getReceivers().size() == 9;
}
return null;
}).when(alertManagerConfigFacade).update(anyObject());
}).when(alertManagerConfigFacade).update(Mockito.any());
alertManagerConfiguration.restoreFromBackup();
AlertManagerConfig alertManagerConfig = alertManagerConfigController.read();
assert alertManagerConfig.getRoute().getRoutes().size() == 15;
Expand All @@ -266,7 +263,7 @@ public void testBackupToDb()
assert alertManagerConfig.getReceivers().size() == 5;
}
return null;
}).when(alertManagerConfigFacade).save(anyObject());
}).when(alertManagerConfigFacade).save(Mockito.any());
Mockito.when(alertManagerConfigFacade.getLatest()).thenReturn(Optional.empty());
alertManagerConfiguration.restoreFromBackup();
}
Expand Down
2 changes: 1 addition & 1 deletion hopsworks-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@
<!-- Test dependencies -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<artifactId>mockito-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand Down
12 changes: 7 additions & 5 deletions hopsworks-api/src/test/java/io/hops/hopsworks/api/featurestore/commit/TestCommitBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import java.util.ArrayList;
import java.util.HashSet;

import static org.mockito.Matchers.*;
import static org.mockito.Mockito.verify;

public class TestCommitBuilder {
Expand All @@ -48,22 +47,25 @@ public class TestCommitBuilder {
@Before
public void setup() {
featureGroupCommitController = Mockito.mock(FeatureGroupCommitController.class);
Mockito.when(featureGroupCommitController.getCommitDetails(anyInt(), anyInt(), anyInt(), anySet(), anySet()))
Mockito.when(featureGroupCommitController.getCommitDetails(
Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anySet(), Mockito.anySet()))
.thenReturn(new AbstractFacade.CollectionInfo(0L, new ArrayList()));

//throws exception if limit or offset is null
Mockito.when(featureGroupCommitController
.getCommitDetails(anyInt(), anyInt(), (Integer) isNull(),anySet(), anySet()))
.getCommitDetails(
Mockito.anyInt(), Mockito.anyInt(), Mockito.isNull(),Mockito.anySet(), Mockito.anySet()))
.thenThrow(new NullPointerException());
Mockito.when(featureGroupCommitController
.getCommitDetails(anyInt(), (Integer) isNull(), anyInt(), anySet(), anySet()))
.getCommitDetails(
Mockito.anyInt(), Mockito.isNull(), Mockito.anyInt(), Mockito.anySet(), Mockito.anySet()))
.thenThrow(new NullPointerException());

commitBuilder = new CommitBuilder(featureGroupCommitController);

mockUriInfo = Mockito.mock(UriInfo.class);
UriBuilder mockUriBuilder = Mockito.mock(UriBuilder.class);
Mockito.when(mockUriBuilder.path(anyString()))
Mockito.when(mockUriBuilder.path(Mockito.anyString()))
.thenReturn(mockUriBuilder);
Mockito.when(mockUriInfo.getBaseUriBuilder())
.thenReturn(mockUriBuilder);
Expand Down
2 changes: 1 addition & 1 deletion hopsworks-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@
<!-- Test dependencies -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<artifactId>mockito-core</artifactId>
</dependency>
<!-- enhance functional java - Try construct - used by provenance -->
<dependency>
Expand Down
16 changes: 16 additions & 0 deletions ...opsworks/common/featurestore/transformationFunction/TransformationFunctionController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ public class TransformationFunctionController {

private static final String TRANSFORMATIONFUNCTIONS_FOLDER = "transformation_functions";

public TransformationFunctionController() {
}

// for testing
public TransformationFunctionController(TransformationFunctionFacade transformationFunctionFacade) {
this.transformationFunctionFacade = transformationFunctionFacade;
}

public TransformationFunction register(Users user, Project project, Featurestore featurestore,
TransformationFunctionDTO transformationFunctionDTO)
throws FeaturestoreException, IOException {
Expand Down Expand Up @@ -208,6 +216,14 @@ public void delete(Project project, Featurestore featurestore, Users user, Integ
new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRANSFORMATION_FUNCTION_DOES_NOT_EXIST,
Level.FINE, "Could not find transformation function with ID" + transformationFunctionId));

// Check if trying to delete built in transformation function
if (FeaturestoreConstants.BUILT_IN_TRANSFORMATION_FUNCTION_NAMES.contains(transformationFunction.getName())
&& transformationFunction.getVersion() == 1) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ERROR_DELETING_TRANSFORMERFUNCTION, Level.FINE,
"Deleting built-in transformation function `" + transformationFunction.getName() + "` with version 1 is not " +
"allowed. Create a new version instead.");
}

DistributedFileSystemOps udfso = null;
try {
udfso = dfs.getDfsOps(hdfsUsersController.getHdfsUserName(project, user));
Expand Down
52 changes: 25 additions & 27 deletions hopsworks-common/src/test/io/hops/hopsworks/common/elastic/TestElasticClientController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@
import org.junit.Test;
import org.mockito.Mockito;

import static org.mockito.Matchers.any;

public class TestElasticClientController {

private ElasticHits.Handler handler;
Expand All @@ -46,7 +44,7 @@ public class TestElasticClientController {
@Before
public void setup() {
handler = Mockito.mock(ElasticHits.Handler.class);
Mockito.stub(handler.apply(any())).toReturn(new Try.Success(null));
Mockito.when(handler.apply(Mockito.any())).thenReturn(new Try.Success(null));
client = Mockito.spy(new ElasticClientController());

BytesReference create = new BytesArray("{\n" +
Expand Down Expand Up @@ -112,23 +110,23 @@ public void testScrollingWithScrollId() throws ElasticException {

//region Mock
Mockito.doReturn(null)
.when(client).baseSearch(any());
.when(client).baseSearch(Mockito.any());

Mockito.doReturn(null)
.when(client).searchScrollingInt(any());
.when(client).searchScrollingInt(Mockito.any());

Mockito.doReturn(null)
.when(client).clearScrollingContext(any());
.when(client).clearScrollingContext(Mockito.any());
//endregion

//act
Pair<Long, Try<Object>> result = client.scrolling(response, handler, request);

//assert
Assert.assertEquals(1, result.getValue0().intValue());
Mockito.verify(client, Mockito.times(0)).baseSearch(any());
Mockito.verify(client, Mockito.times(0)).searchScrollingInt(any());
Mockito.verify(client, Mockito.times(1)).clearScrollingContext(any());
Mockito.verify(client, Mockito.times(0)).baseSearch(Mockito.any());
Mockito.verify(client, Mockito.times(0)).searchScrollingInt(Mockito.any());
Mockito.verify(client, Mockito.times(1)).clearScrollingContext(Mockito.any());
}

@Test
Expand All @@ -145,23 +143,23 @@ public void testScrollingWithoutScrollId() throws ElasticException {

//region Mock
Mockito.doReturn(null)
.when(client).baseSearch(any());
.when(client).baseSearch(Mockito.any());

Mockito.doReturn(null)
.when(client).searchScrollingInt(any());
.when(client).searchScrollingInt(Mockito.any());

Mockito.doReturn(null)
.when(client).clearScrollingContext(any());
.when(client).clearScrollingContext(Mockito.any());
//endregion

//act
Pair<Long, Try<Object>> result = client.scrolling(response, handler, request);

//assert
Assert.assertEquals(1, result.getValue0().intValue());
Mockito.verify(client, Mockito.times(0)).baseSearch(any());
Mockito.verify(client, Mockito.times(0)).searchScrollingInt(any());
Mockito.verify(client, Mockito.times(0)).clearScrollingContext(any());
Mockito.verify(client, Mockito.times(0)).baseSearch(Mockito.any());
Mockito.verify(client, Mockito.times(0)).searchScrollingInt(Mockito.any());
Mockito.verify(client, Mockito.times(0)).clearScrollingContext(Mockito.any());
}

@Test
Expand All @@ -178,23 +176,23 @@ public void testScrollingHasLeftoverWithScrollId() throws ElasticException {

//region Mock
Mockito.doReturn(null)
.when(client).baseSearch(any());
.when(client).baseSearch(Mockito.any());

Mockito.doReturn(createResponse(new SearchHit[] { readHit }, totalHits, ""))
.when(client).searchScrollingInt(any());
.when(client).searchScrollingInt(Mockito.any());

Mockito.doReturn(null)
.when(client).clearScrollingContext(any());
.when(client).clearScrollingContext(Mockito.any());
//endregion

//act
Pair<Long, Try<Object>> result = client.scrolling(response, handler, request);

//assert
Assert.assertEquals(2, result.getValue0().intValue());
Mockito.verify(client, Mockito.times(0)).baseSearch(any());
Mockito.verify(client, Mockito.times(1)).searchScrollingInt(any());
Mockito.verify(client, Mockito.times(1)).clearScrollingContext(any());
Mockito.verify(client, Mockito.times(0)).baseSearch(Mockito.any());
Mockito.verify(client, Mockito.times(1)).searchScrollingInt(Mockito.any());
Mockito.verify(client, Mockito.times(1)).clearScrollingContext(Mockito.any());
}

@Test
Expand All @@ -211,22 +209,22 @@ public void testScrollingHasLeftoverWithoutScrollId() throws ElasticException {

//region Mock
Mockito.doReturn(createResponse(new SearchHit[] { createHit }, totalHits, ""))
.when(client).baseSearch(any());
.when(client).baseSearch(Mockito.any());

Mockito.doReturn(createResponse(new SearchHit[] { readHit }, totalHits, ""))
.when(client).searchScrollingInt(any());
.when(client).searchScrollingInt(Mockito.any());

Mockito.doReturn(null)
.when(client).clearScrollingContext(any());
.when(client).clearScrollingContext(Mockito.any());
//endregion

//act
Pair<Long, Try<Object>> result = client.scrolling(response, handler, request);

//assert
Assert.assertEquals(2, result.getValue0().intValue());
Mockito.verify(client, Mockito.times(1)).baseSearch(any());
Mockito.verify(client, Mockito.times(1)).searchScrollingInt(any());
Mockito.verify(client, Mockito.times(1)).clearScrollingContext(any());
Mockito.verify(client, Mockito.times(1)).baseSearch(Mockito.any());
Mockito.verify(client, Mockito.times(1)).searchScrollingInt(Mockito.any());
Mockito.verify(client, Mockito.times(1)).clearScrollingContext(Mockito.any());
}
}
16 changes: 7 additions & 9 deletions ...ommon/src/test/io/hops/hopsworks/common/featurestore/query/TestConstructorController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import io.hops.hopsworks.common.featurestore.query.filter.Filter;
import io.hops.hopsworks.common.featurestore.query.filter.FilterController;
import io.hops.hopsworks.common.featurestore.query.filter.FilterLogic;
import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.SqlFilterLogic;
import io.hops.hopsworks.common.featurestore.query.join.Join;
import io.hops.hopsworks.common.featurestore.query.join.JoinController;
import io.hops.hopsworks.common.featurestore.query.join.JoinDTO;
Expand All @@ -36,6 +35,7 @@
import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup;
import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.TimeTravelFormat;
import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.SqlCondition;
import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.SqlFilterLogic;
import io.hops.hopsworks.persistence.entity.project.Project;
import io.hops.hopsworks.persistence.entity.user.Users;
import org.apache.calcite.sql.JoinType;
Expand All @@ -58,8 +58,6 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.mockito.Matchers.eq;

public class TestConstructorController {

private Featurestore fs;
Expand Down Expand Up @@ -913,7 +911,7 @@ public void testMakeOfflineQuery_hiveQuery() throws Exception {
Query query = new Query("fs1", "project_fs1", fgHudi, "fg0", availableLeft, availableLeft);
query.setOrderByFeatures(availableLeft);
query.setHiveEngine(true);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(eq(availableLeft))).thenReturn(availableLeft);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(availableLeft)).thenReturn(availableLeft);
String actual = constructorController.makeOfflineQuery(query);
String expected = "SELECT `fg0`.`ft1`\n" +
"FROM `fs1`.`fgHudi_1` `fg0`\n" +
Expand Down Expand Up @@ -942,8 +940,8 @@ public void testMakeOfflineQuery_hiveQueryNested() throws Exception {

List<Feature> allFeatures = new ArrayList<>(availableLeft);
allFeatures.addAll(availableRight);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(eq(allFeatures))).thenReturn(allFeatures);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(eq(availableRight))).thenReturn(availableRight);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(allFeatures)).thenReturn(allFeatures);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(availableRight)).thenReturn(availableRight);

String actual = constructorController.makeOfflineQuery(leftQuery);
String expected = "SELECT `fg0`.`ft1`, `fg1`.`ft1`\n" +
Expand All @@ -960,7 +958,7 @@ public void testMakeOfflineQuery_sparkQuery() throws Exception {
Query query = new Query("fs1", "project_fs1", fgHudi, "fg0", availableLeft, availableLeft);
query.setOrderByFeatures(availableLeft);
query.setHiveEngine(false);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(eq(availableLeft))).thenReturn(availableLeft);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(availableLeft)).thenReturn(availableLeft);
String actual = constructorController.makeOfflineQuery(query);
String expected = "SELECT `fg0`.`ft1`\n" +
"FROM `fg0` `fg0`\n" +
Expand Down Expand Up @@ -988,8 +986,8 @@ public void testMakeOfflineQuery_sparkQueryNested() throws Exception {
leftQuery.setOrderByFeatures(availableLeft);
List<Feature> allFeatures = new ArrayList<>(availableLeft);
allFeatures.addAll(availableRight);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(eq(allFeatures))).thenReturn(allFeatures);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(eq(availableRight))).thenReturn(availableRight);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(allFeatures)).thenReturn(allFeatures);
Mockito.when(cachedFeaturegroupController.dropHudiSpecFeatures(availableRight)).thenReturn(availableRight);

String actual = constructorController.makeOfflineQuery(leftQuery);
String expected = "SELECT `fg0`.`ft1`, `fg1`.`ft1`\n" +
Expand Down
Loading

0 comments on commit f835c26

Please sign in to comment.