Skip to content
This repository has been archived by the owner on Jan 11, 2019. It is now read-only.

Commit

Permalink
AVRO-1887 Integrate Yetus and fix tests (apache#353)
Browse files Browse the repository at this point in the history
* AVRO-1887 Setup precommit checks via Apache Yetus

Signed-off-by: sacharya <[email protected]>

* AVRO-1887 Integrate Yetus

Add a precommit hook using Apache Yetus that will invoke the test
suite of the different languages

- Disable Ruby integration tests tests
- Fix Flaky Java Datetime test
  When the milsecons would have trailing zero, it would get trimmed
- Align the order of imports
  • Loading branch information
Fokko authored Nov 7, 2018
1 parent fd26721 commit 25032e1
Show file tree
Hide file tree
Showing 108 changed files with 1,658 additions and 1,888 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ test-output
/dist
/lang/java/compiler/nbactions.xml
/lang/java/compiler/nb-configuration.xml
/lang/java/compiler/nbproject/
/lang/java/compiler/nbproject/
34 changes: 34 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

sudo: required

services:
- docker

language: generic

before_install:
- sudo apt-get -q update
- sudo apt-get -q install --no-install-recommends -y curl git gnupg-agent locales pinentry-curses pkg-config rsync software-properties-common
- sudo apt-get -q clean
- sudo rm -rf /var/lib/apt/lists/*
- curl -L https://www-us.apache.org/dist/yetus/0.8.0/yetus-0.8.0-bin.tar.gz | tar xvz -C /tmp/

script: /tmp/yetus-0.8.0/bin/test-patch --plugins=buildtest --user-plugins=share/precommit/ --run-tests --empty-patch --docker --dockerfile=share/docker/Dockerfile --dirty-workspace --verbose=true
4 changes: 4 additions & 0 deletions README.txt → README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
[![Build Status](https://travis-ci.org/apache/avro.svg?branch=master)](https://travis-ci.org/apache/avro)

# Apache Avro™

Apache Avro™ is a data serialization system.

Learn more about Avro, please visit our website at:
Expand Down
21 changes: 12 additions & 9 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,39 +40,42 @@ do
test)
# run lang-specific tests
(cd lang/java; ./build.sh test)

# create interop test data
mkdir -p build/interop/data
(cd lang/java/avro; mvn -B -P interop-data-generate generate-resources)

# install java artifacts required by other builds and interop tests
mvn install -DskipTests
mvn -B install -DskipTests
(cd lang/py; ./build.sh test)
(cd lang/py3; ./build.sh test)
(cd lang/c; ./build.sh test)
(cd lang/c++; ./build.sh test)
#(cd lang/c++; ./build.sh test)
(cd lang/csharp; ./build.sh test)
(cd lang/js; ./build.sh test)
(cd lang/ruby; ./build.sh test)
(cd lang/php; ./build.sh test)
(cd lang/perl; ./build.sh test)

# create interop test data
mkdir -p build/interop/data
(cd lang/java/avro; mvn -P interop-data-generate generate-resources)
(cd lang/py; ant interop-data-generate)
(cd lang/c; ./build.sh interop-data-generate)
#(cd lang/c++; make interop-data-generate)
(cd lang/ruby; rake generate_interop)
(cd lang/php; ./build.sh interop-data-generate)

# run interop data tests
(cd lang/java; mvn test -P interop-data-test)
(cd lang/java; mvn -B test -P interop-data-test)
(cd lang/py; ant interop-data-test)
(cd lang/c; ./build.sh interop-data-test)
#(cd lang/c++; make interop-data-test)
(cd lang/ruby; rake interop)
(cd lang/php; ./build.sh test-interop)

# java needs to package the jars for the interop rpc tests
(cd lang/java; mvn package -DskipTests)
(cd lang/java; mvn -B package -DskipTests)

# run interop rpc test
/bin/bash share/test/interop/bin/test_rpc_interop.sh
./share/test/interop/bin/test_rpc_interop.sh
;;

dist)
Expand Down Expand Up @@ -165,7 +168,7 @@ do
rm -rf build dist
(cd doc; ant clean)

(mvn clean)
(mvn -B clean)
rm -rf lang/java/*/userlogs/
rm -rf lang/java/*/dependency-reduced-pom.xml

Expand Down
1 change: 0 additions & 1 deletion lang/java/avro/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -194,4 +194,3 @@
</dependencies>

</project>

54 changes: 0 additions & 54 deletions lang/java/avro/src/test/java/org/apache/avro/AvroTestUtil.java

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@
*/
package org.apache.avro;

import org.apache.avro.specific.SpecificData;
import org.apache.avro.message.BinaryMessageEncoder;
import org.apache.avro.message.BinaryMessageDecoder;
import org.apache.avro.message.BinaryMessageEncoder;
import org.apache.avro.message.SchemaStore;
import org.apache.avro.specific.SpecificData;

@org.apache.avro.specific.AvroGenerated
public class FooBarSpecificRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@
*/
package org.apache.avro;

import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;

/**
* Generates file with objects of a specific schema(that doesn't contain nesting
* of arrays and maps) with random data. This is only for testing.
Expand Down
6 changes: 1 addition & 5 deletions lang/java/avro/src/test/java/org/apache/avro/RandomData.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,7 @@

import java.io.File;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.*;

import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.File;
import java.io.IOException;
import java.util.*;

import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.file.FileReader;
Expand Down
22 changes: 10 additions & 12 deletions lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,14 @@
import java.util.Random;

import junit.framework.Assert;

import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.FileReader;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileStream;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.file.SeekableFileInput;
import org.apache.avro.file.Syncable;
import org.apache.avro.file.*;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DatumReader;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
Expand All @@ -52,7 +47,10 @@ public class TestDataFile {
private static final Logger LOG =
LoggerFactory.getLogger(TestDataFile.class);

CodecFactory codec = null;
@Rule
public TemporaryFolder DIR = new TemporaryFolder();

private final CodecFactory codec;
public TestDataFile(CodecFactory codec) {
this.codec = codec;
LOG.info("Running with codec: " + codec);
Expand All @@ -78,8 +76,8 @@ public static List<Object[]> codecs() {
Integer.parseInt(System.getProperty("test.count", "200"));
private static final boolean VALIDATE =
!"false".equals(System.getProperty("test.validate", "true"));
private static final File DIR
= new File(System.getProperty("test.dir", "/tmp"));


private static final long SEED = System.currentTimeMillis();
private static final String SCHEMA_JSON =
"{\"type\": \"record\", \"name\": \"Test\", \"fields\": ["
Expand All @@ -88,7 +86,7 @@ public static List<Object[]> codecs() {
private static final Schema SCHEMA = new Schema.Parser().parse(SCHEMA_JSON);

private File makeFile() {
return new File(DIR, "test-" + codec + ".avro");
return new File(DIR.getRoot().getPath(), "test-" + codec + ".avro");
}

@Test public void runTestsInOrder() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
*/
package org.apache.avro;

import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;

import java.io.File;
import java.io.IOException;
Expand All @@ -29,7 +29,9 @@
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
Expand All @@ -41,8 +43,11 @@ public class TestDataFileConcat {
private static final Logger LOG =
LoggerFactory.getLogger(TestDataFileConcat.class);

CodecFactory codec = null;
CodecFactory codec2 = null;
@Rule
public TemporaryFolder DIR = new TemporaryFolder();

CodecFactory codec;
CodecFactory codec2;
boolean recompress;
public TestDataFileConcat(CodecFactory codec, CodecFactory codec2, Boolean recompress) {
this.codec = codec;
Expand Down Expand Up @@ -80,8 +85,7 @@ public static List<Object[]> codecs() {
Integer.parseInt(System.getProperty("test.count", "200"));
private static final boolean VALIDATE =
!"false".equals(System.getProperty("test.validate", "true"));
private static final File DIR
= new File(System.getProperty("test.dir", "/tmp"));

private static final long SEED = System.currentTimeMillis();

private static final String SCHEMA_JSON =
Expand All @@ -93,7 +97,7 @@ public static List<Object[]> codecs() {
private static final Schema SCHEMA = new Schema.Parser().parse(SCHEMA_JSON);

private File makeFile(String name) {
return new File(DIR, "test-" + name + ".avro");
return new File(DIR.getRoot().getPath(), "test-" + name + ".avro");
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
*/
package org.apache.avro;

import static org.junit.Assert.*;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.avro.Schema.Type;
import org.apache.avro.file.DataFileConstants;
import org.apache.avro.file.DataFileReader;
Expand All @@ -30,14 +33,9 @@
import org.apache.avro.util.Utf8;
import org.junit.Test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;

public class TestDataFileCorruption {

private static final File DIR
= new File(System.getProperty("test.dir", "/tmp"));
private static final File DIR = new File( "/tmp");

private File makeFile(String name) {
return new File(DIR, "test-" + name + ".avro");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,22 @@
*/
package org.apache.avro;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.UUID;

import org.apache.avro.Schema.Type;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.util.Utf8;
import org.junit.Test;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class TestDataFileCustomSync {
private byte[] createDataFile(byte[] sync) throws IOException {
Schema schema = Schema.create(Type.STRING);
Expand Down
Loading

0 comments on commit 25032e1

Please sign in to comment.