Skip to content

Commit

Permalink
Merge jobs tables and other database improvements (Netflix#651)
Browse files Browse the repository at this point in the history 
* Database upgrade scripts for 3.3.0

- Stashing scripts for now
- Stashing some more 3.3.0 changes
- Swith to UTF-8 (variable length) and add triggers for file references
- Fixing trigger procedure for files and other modifications
- Renamed the upgrade script to reflect expected version number
- Modify sql schema to still include tags columns AND have normalized tag tables for analytics
- Add more logging
- Adding hsqldb and postgresql scripts for 3.3.0
- Add uuid field to all tables for entity equals and hash code
- Remove UUID column and use unique_id column for tables that need it. Fix Postgres upgrade script errors that were missed

* Upgrade to Spring Cloud Dalston SR4

* Move criterion entities to their own table

* Fixes for optionals and checks of optionals

* Rename entity classes for more clarity. Break out the id field into a true base entity in case we want tables without audit fields

* Remove audit fields from criterion entity as they're tied to a specific job and will always have the same data it's just a waste of space

* Add grouping and groupingInstance to DTO's. Deprecate old constructors for builders. Move command args to internal List<String> representation

* Adding tests for mapping between DTO and entities. Added grouping and groupingInstance to mappings.

* Add grouping and groupingInstance as search parameters. Move command args in database to separate table to support list of command args going forward

* Removing UUID default from Postgres scripts

* Add Identity Generation type to the entities to fix weird behavior in Postgresql when AUTO is on

* Ensure database connection encodings for MySQL are UTF-8

* Upgrade some out of date hardcoded dependencies (not from boms)

* Upgrade spectator to 0.60.0 and fix deprecation warnings this caused
  • Loading branch information
@tgianos
tgianos authored Nov 28, 2017
1 parent c0998b1 commit b5ab6a1
Show file tree
Hide file tree
Showing 175 changed files with 15,979 additions and 7,477 deletions.
21 changes: 14 additions & 7 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ allprojects {
maven {
url "http://repo.spring.io/milestone"
}
maven {
url "http://repo.spring.io/libs-snapshot"
}
}
}

Expand Down Expand Up @@ -71,7 +74,7 @@ configure(javaProjects) {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${spring_cloud_version}"
}
dependencies {
dependency("cglib:cglib-nodep:3.2.4")
dependency("cglib:cglib-nodep:3.2.5")
dependency("com.github.fge:json-patch:1.9")
dependency("com.github.springtestdbunit:spring-test-dbunit:1.3.0")
dependencySet(group: "com.squareup.retrofit2", version: "2.1.0") {
Expand All @@ -82,7 +85,7 @@ configure(javaProjects) {
entry "servo-core"
entry "servo-internal"
}
dependencySet(group: "com.netflix.spectator", version: "0.56.0") {
dependencySet(group: "com.netflix.spectator", version: "0.60.0") {
entry "spectator-api"
entry "spectator-reg-servo"
}
Expand All @@ -91,20 +94,24 @@ configure(javaProjects) {
entry "springfox-swagger-ui"
entry "springfox-bean-validators"
}
dependency("commons-validator:commons-validator:1.5.1")
dependency("commons-validator:commons-validator:1.6")
dependency("net.sf.jtidy:jtidy:r938")
dependency("org.apache.commons:commons-configuration2:2.1")
dependency("org.apache.commons:commons-configuration2:2.2")
dependency("org.apache.commons:commons-exec:1.3")
dependency("org.bitbucket.b_c:jose4j:0.5.4")
dependency("org.dbunit:dbunit:2.5.3")
dependencySet(group: "org.jruby", version:"9.1.12.0") {
dependency("org.bitbucket.b_c:jose4j:0.6.2")
dependency("org.dbunit:dbunit:2.5.4")
dependencySet(group: "org.jruby", version:"9.1.14.0") {
entry "jruby-complete"
entry "jruby-core"
entry "jruby-stdlib"
}
dependency("org.opensaml:opensaml:2.6.4")
dependency("org.python:jython-standalone:2.7.1")
dependency("org.springframework.security.extensions:spring-security-saml2-core:1.0.2.RELEASE")

// TODO: These are here as part of waiting for bug fix release https://jira.spring.io/browse/DATAJPA-1185
dependency("org.springframework.data:spring-data-commons:1.13.8.RELEASE")
dependency("org.springframework.data:spring-data-jpa:1.11.9.RELEASE")
}
}

Expand Down
10 changes: 5 additions & 5 deletions genie-client/src/main/java/com/netflix/genie/client/BaseGenieClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
* @author amsharma
* @since 3.0.0
*/
public abstract class BaseGenieClient {
abstract class BaseGenieClient {

private Retrofit retrofit;
private ObjectMapper mapper;
Expand All @@ -57,7 +57,7 @@ public abstract class BaseGenieClient {
* @param genieNetworkConfiguration A configuration object that provides network settings for HTTP calls.
* @throws GenieClientException If there is any problem creating the constructor.
*/
public BaseGenieClient(
BaseGenieClient(
@NotEmpty final String url,
@Nullable final List<Interceptor> interceptors,
@Nullable final GenieNetworkConfiguration genieNetworkConfiguration
Expand Down Expand Up @@ -120,15 +120,15 @@ private void addConfigParamsFromConfig(
* @param location The location string in the header.
* @return The id of the entity embedded in the location.
*/
protected String getIdFromLocation(final String location) {
String getIdFromLocation(final String location) {
return location.substring(location.lastIndexOf("/") + 1);
}

protected <T> T getService(final Class<T> clazz) {
<T> T getService(final Class<T> clazz) {
return this.retrofit.create(clazz);
}

protected <T> T treeToValue(final JsonNode node, final Class<T> clazz) throws IOException {
<T> T treeToValue(final JsonNode node, final Class<T> clazz) throws IOException {
return this.mapper.treeToValue(node, clazz);
}
}
98 changes: 94 additions & 4 deletions genie-client/src/main/java/com/netflix/genie/client/JobClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,11 +156,13 @@ public void writeTo(final BufferedSink sink) throws IOException {
* @throws IOException For Network and other IO issues.
*/
public List<JobSearchResult> getJobs() throws IOException, GenieClientException {
return this.getJobs(null, null, null, null, null, null, null, null, null, null, null, null, null);
return this.getJobs(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
}

/**
* Method to get a list of all the jobs from Genie for the query parameters specified.
* <p>
* Deprecated: For new search fields
*
* @param id id for job
* @param name name of job (can be a SQL-style pattern such as HIVE%)
Expand All @@ -176,9 +178,27 @@ public List<JobSearchResult> getJobs() throws IOException, GenieClientException
* @param minFinished The time which the job had to finish after in order to be return (inclusive)
* @param maxFinished The time which the job had to finish before in order to be returned (exclusive)
* @return A list of jobs.
* @throws GenieClientException If the response recieved is not 2xx.
* @throws GenieClientException If the response received is not 2xx.
* @throws IOException For Network and other IO issues.
* @see #getJobs(
*String,
* String,
* String,
* Set,
* Set,
* String,
* String,
* String,
* String,
* Long,
* Long,
* Long,
* Long,
* String,
* String
*)
*/
@Deprecated
public List<JobSearchResult> getJobs(
final String id,
final String name,
Expand All @@ -195,7 +215,75 @@ public List<JobSearchResult> getJobs(
final Long maxFinished
) throws IOException, GenieClientException {

final JsonNode jnode = jobService.getJobs(
final JsonNode jnode = this.jobService.getJobs(
id,
name,
user,
statuses,
tags,
clusterName,
clusterId,
commandName,
commandId,
minStarted,
maxStarted,
minFinished,
maxFinished,
null,
null
).execute().body()
.get("_embedded")
.get("jobSearchResultList");

final List<JobSearchResult> jobList = new ArrayList<>();
for (final JsonNode objNode : jnode) {
final JobSearchResult jobSearchResult = this.treeToValue(objNode, JobSearchResult.class);
jobList.add(jobSearchResult);
}
return jobList;
}

/**
* Method to get a list of all the jobs from Genie for the query parameters specified.
*
* @param id id for job
* @param name name of job (can be a SQL-style pattern such as HIVE%)
* @param user user who submitted job
* @param statuses statuses of jobs to find
* @param tags tags for the job
* @param clusterName the name of the cluster
* @param clusterId the id of the cluster
* @param commandName the name of the command run by the job
* @param commandId the id of the command run by the job
* @param minStarted The time which the job had to start after in order to be return (inclusive)
* @param maxStarted The time which the job had to start before in order to be returned (exclusive)
* @param minFinished The time which the job had to finish after in order to be return (inclusive)
* @param maxFinished The time which the job had to finish before in order to be returned (exclusive)
* @param grouping The grouping the job should be a member of
* @param groupingInstance The grouping instance the job should be a member of
* @return A list of jobs.
* @throws GenieClientException If the response received is not 2xx.
* @throws IOException For Network and other IO issues.
*/
public List<JobSearchResult> getJobs(
@Nullable final String id,
@Nullable final String name,
@Nullable final String user,
@Nullable final Set<String> statuses,
@Nullable final Set<String> tags,
@Nullable final String clusterName,
@Nullable final String clusterId,
@Nullable final String commandName,
@Nullable final String commandId,
@Nullable final Long minStarted,
@Nullable final Long maxStarted,
@Nullable final Long minFinished,
@Nullable final Long maxFinished,
@Nullable final String grouping,
@Nullable final String groupingInstance
) throws IOException, GenieClientException {

final JsonNode jnode = this.jobService.getJobs(
id,
name,
user,
Expand All @@ -208,7 +296,9 @@ public List<JobSearchResult> getJobs(
minStarted,
maxStarted,
minFinished,
maxFinished
maxFinished,
grouping,
groupingInstance
).execute().body()
.get("_embedded")
.get("jobSearchResultList");
Expand Down
36 changes: 19 additions & 17 deletions genie-client/src/main/java/com/netflix/genie/client/apis/JobService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,8 @@ public interface JobService {
/**
* Submit a job with attachments.
*
* @param request A JobRequest object containing all the details needed to run the job.
* @param request A JobRequest object containing all the details needed to run the job.
* @param attachments A list of all the attachment files to be sent to the server.
*
* @return A callable object.
*/
@Multipart
Expand All @@ -79,20 +78,21 @@ Call<Void> submitJobWithAttachments(
/**
* Method to get all jobs from Genie.
*
* @param id id for job
* @param name name of job (can be a SQL-style pattern such as HIVE%)
* @param user user who submitted job
* @param statuses statuses of jobs to find
* @param tags tags for the job
* @param clusterName the name of the cluster
* @param clusterId the id of the cluster
* @param commandName the name of the command run by the job
* @param commandId the id of the command run by the job
* @param minStarted The time which the job had to start after in order to be return (inclusive)
* @param maxStarted The time which the job had to start before in order to be returned (exclusive)
* @param minFinished The time which the job had to finish after in order to be return (inclusive)
* @param maxFinished The time which the job had to finish before in order to be returned (exclusive)
*
* @param id id for job
* @param name name of job (can be a SQL-style pattern such as HIVE%)
* @param user user who submitted job
* @param statuses statuses of jobs to find
* @param tags tags for the job
* @param clusterName the name of the cluster
* @param clusterId the id of the cluster
* @param commandName the name of the command run by the job
* @param commandId the id of the command run by the job
* @param minStarted The time which the job had to start after in order to be return (inclusive)
* @param maxStarted The time which the job had to start before in order to be returned (exclusive)
* @param minFinished The time which the job had to finish after in order to be return (inclusive)
* @param maxFinished The time which the job had to finish before in order to be returned (exclusive)
* @param grouping The grouping the job should be a member of
* @param groupingInstance The grouping instance the job should be a member of
* @return A callable object.
*/
@GET(JOBS_URL_SUFFIX)
Expand All @@ -109,7 +109,9 @@ Call<JsonNode> getJobs(
@Query("minStarted") final Long minStarted,
@Query("maxStarted") final Long maxStarted,
@Query("minFinished") final Long minFinished,
@Query("maxFinished") final Long maxFinished
@Query("maxFinished") final Long maxFinished,
@Query("grouping") final String grouping,
@Query("groupingInstance") final String groupingInstance
);

/**
Expand Down
Loading

0 comments on commit b5ab6a1

Please sign in to comment.