Skip to content

Commit

Permalink
Move to poetry, github actions, and python 3.6 or greater
Browse files Browse the repository at this point in the history
  • Loading branch information
EntilZha committed Apr 22, 2020
1 parent c2c7789 commit e14cfd5
Show file tree
Hide file tree
Showing 19 changed files with 871 additions and 163 deletions.
41 changes: 41 additions & 0 deletions .github/workflows/pythonpackage.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python package

on:
push:
branches: [ master, dev ]
pull_request:
branches: [ master ]

jobs:
build:

runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.6, 3.7, pypy3]
#include:
# - python-version: 3.6
# use_pandas: 1
# - python-version: 3.7
# use_pandas: 1
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip poetry
poetry install
apt install pandoc
- name: Pylint
run: poetry run pylint functional
- name: black
run: poetry run black --check functional
- name: Test with pytest
run: |
poetry run pytest --cov=functional
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,5 @@ target/
functional/test/data/tmp
.ipynb_checkpoints/
.mypy_cache/
.vscode/
.python-version
56 changes: 0 additions & 56 deletions .travis.yml

This file was deleted.

4 changes: 1 addition & 3 deletions functional/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,12 @@
Package for creating data pipelines, LINQ-style data analysis, and functional programming. Imports
the primary entrypoint at streams.seq
"""
from __future__ import absolute_import

from functional.streams import seq, pseq

__author__ = "Pedro Rodriguez"
__copyright__ = "Copyright 2019, Pedro Rodriguez"
__license__ = "MIT"
__version__ = "1.3.0"
__version__ = "1.4.0"
__maintainer__ = "Pedro Rodriguez"
__email__ = "[email protected]"
__status__ = "Production"
23 changes: 7 additions & 16 deletions functional/io.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,10 @@
from __future__ import absolute_import
import gzip
import io
import sys
import builtins

from future import builtins
import six


if six.PY2:
WRITE_MODE = 'wb'
else:
WRITE_MODE = 'wt'
WRITE_MODE = 'wt'


class ReusableFile(object):
Expand Down Expand Up @@ -133,7 +127,7 @@ def __init__(self, path, delimiter=None, mode='rt', buffering=-1, compresslevel=
newline=newline)

def __iter__(self):
if six.PY2 or '__pypy__' in sys.builtin_module_names:
if '__pypy__' in sys.builtin_module_names:
import bz2file as bz2 # pylint: disable=import-error
else:
import bz2
Expand All @@ -145,7 +139,7 @@ def __iter__(self):
yield line

def read(self):
if six.PY2 or '__pypy__' in sys.builtin_module_names:
if '__pypy__' in sys.builtin_module_names:
import bz2file as bz2 # pylint: disable=import-error
else:
import bz2
Expand Down Expand Up @@ -220,11 +214,8 @@ def universal_write_open(path, mode, buffering=-1, encoding=None, errors=None, n
return builtins.open(path, mode=mode, buffering=buffering, encoding=encoding, errors=errors,
newline=newline)
elif compression in ('gz', 'gzip'):
if six.PY2:
return gzip.open(path, mode=mode, compresslevel=compresslevel)
else:
return gzip.open(path, mode=mode, compresslevel=compresslevel,
errors=errors, newline=newline, encoding=encoding)
return gzip.open(path, mode=mode, compresslevel=compresslevel,
errors=errors, newline=newline, encoding=encoding)
elif compression in ('lzma', 'xz'):
try:
import lzma
Expand All @@ -233,7 +224,7 @@ def universal_write_open(path, mode, buffering=-1, encoding=None, errors=None, n
return lzma.open(path, mode=mode, format=format, check=check, preset=preset,
filters=filters, encoding=encoding, errors=errors, newline=newline)
elif compression == 'bz2':
if six.PY2 or '__pypy__' in sys.builtin_module_names:
if '__pypy__' in sys.builtin_module_names:
import bz2file as bz2 # pylint: disable=import-error
else:
import bz2
Expand Down
2 changes: 0 additions & 2 deletions functional/lineage.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from __future__ import absolute_import

from functional.execution import ExecutionEngine
from functional.transformations import CACHE_T

Expand Down
12 changes: 4 additions & 8 deletions functional/pipeline.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
"""
The pipeline module contains the transformations and actions API of PyFunctional
"""

from __future__ import division, absolute_import

from operator import mul, add
import collections
from functools import reduce, wraps, partial
Expand All @@ -13,7 +10,6 @@
import sqlite3
import re

import six
from tabulate import tabulate

from functional.execution import ExecutionEngine
Expand Down Expand Up @@ -1490,9 +1486,9 @@ def to_file(self, path, delimiter=None, mode='wt', buffering=-1, encoding=None,
compresslevel=compresslevel, format=format, check=check,
preset=preset, filters=filters) as output:
if delimiter:
output.write(six.u(self.make_string(delimiter)))
output.write(self.make_string(delimiter))
else:
output.write(six.u(str(self)))
output.write(str(self))

def to_jsonl(self, path, mode='wb', compression=None):
"""
Expand Down Expand Up @@ -1541,7 +1537,7 @@ def to_csv(self, path, mode=WRITE_MODE, dialect='excel', compression=None,
newline=newline) as output:
csv_writer = csv.writer(output, dialect=dialect, **fmtparams)
for row in self:
csv_writer.writerow([six.u(str(element)) for element in row])
csv_writer.writerow([str(element) for element in row])

def _to_sqlite3_by_query(self, conn, sql):
"""
Expand Down Expand Up @@ -1712,7 +1708,7 @@ def _wrap(value):
return value
if isinstance(value, (dict, set)) or is_namedtuple(value):
return value
elif isinstance(value, collections.Iterable):
elif isinstance(value, collections.abc.Iterable):
try:
if type(value).__name__ == 'DataFrame':
import pandas
Expand Down
8 changes: 2 additions & 6 deletions functional/streams.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
from __future__ import absolute_import

import re
import csv as csvapi
import json as jsonapi
import sqlite3 as sqlite3api

from future import builtins
import six
import builtins

from functional.execution import ExecutionEngine, ParallelExecutionEngine
from functional.pipeline import Sequence
Expand Down Expand Up @@ -201,7 +197,7 @@ def json(self, json_file):
if isinstance(json_input, list):
return self(json_input)
else:
return self(six.viewitems(json_input))
return self(json_input.items())

# pylint: disable=keyword-arg-before-vararg
def sqlite3(self, conn, sql, parameters=None, *args, **kwargs):
Expand Down
2 changes: 0 additions & 2 deletions functional/test/test_functional.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
# pylint: skip-file
from __future__ import absolute_import

import unittest
import array
from collections import namedtuple
Expand Down
19 changes: 6 additions & 13 deletions functional/test/test_streams.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,16 @@
from __future__ import absolute_import

import sqlite3
import unittest
import collections
import sys
import gzip
from platform import system

import six
import lzma

from functional import seq, pseq
from functional.streams import Stream, ParallelStream

try:
import lzma
except ImportError:
from backports import lzma

if six.PY2 or '__pypy__' in sys.builtin_module_names:
if '__pypy__' in sys.builtin_module_names:
import bz2file as bz2 # pylint: disable=import-error
else:
import bz2
Expand Down Expand Up @@ -146,7 +139,7 @@ def test_json(self):
list_test_path = 'functional/test/data/test_list.json'
dict_test_path = 'functional/test/data/test_dict.json'
list_expect = [1, 2, 3, 4, 5]
dict_expect = list(six.viewitems({u'a': 1, u'b': 2, u'c': 3}))
dict_expect = list({u'a': 1, u'b': 2, u'c': 3}.items())

result = self.seq.json(list_test_path).to_list()
self.assertEqual(list_expect, result)
Expand All @@ -167,7 +160,7 @@ def test_gzip_json(self):
list_test_path = 'functional/test/data/test_list.json.gz'
dict_test_path = 'functional/test/data/test_dict.json.gz'
list_expect = [1, 2, 3, 4, 5]
dict_expect = list(six.viewitems({u'a': 1, u'b': 2, u'c': 3}))
dict_expect = list({u'a': 1, u'b': 2, u'c': 3}.items())

result = self.seq.json(list_test_path).to_list()
self.assertEqual(list_expect, result)
Expand All @@ -181,7 +174,7 @@ def test_bz2_json(self):
list_test_path = 'functional/test/data/test_list.json.bz2'
dict_test_path = 'functional/test/data/test_dict.json.bz2'
list_expect = [1, 2, 3, 4, 5]
dict_expect = list(six.viewitems({u'a': 1, u'b': 2, u'c': 3}))
dict_expect = list({u'a': 1, u'b': 2, u'c': 3}.items())

result = self.seq.json(list_test_path).to_list()
self.assertEqual(list_expect, result)
Expand All @@ -195,7 +188,7 @@ def test_xz_json(self):
list_test_path = 'functional/test/data/test_list.json.xz'
dict_test_path = 'functional/test/data/test_dict.json.xz'
list_expect = [1, 2, 3, 4, 5]
dict_expect = list(six.viewitems({u'a': 1, u'b': 2, u'c': 3}))
dict_expect = list({u'a': 1, u'b': 2, u'c': 3}.items())

result = self.seq.json(list_test_path).to_list()
self.assertEqual(list_expect, result)
Expand Down
2 changes: 0 additions & 2 deletions functional/test/test_util.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from __future__ import absolute_import

import unittest
import sys
from collections import namedtuple
Expand Down
Loading

0 comments on commit e14cfd5

Please sign in to comment.