Initial commit

.circleci/config.yml

@@ -0,0 +1,104 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+version: 2
+jobs:
+  test:
+    docker:
+      - image: circleci/python:3.7
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-python3.7-{{ checksum "requirements.txt" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-python3.7-
+
+      - run:
+          name: install dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install -r requirements.txt
+
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-python3.7-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: run tests
+          command: |
+            . venv/bin/activate
+            TESTING=1 pytest -rf test/
+
+      - run:
+          name: check linting
+          command: |
+            . venv/bin/activate
+            flake8 --max-line-length=100 parsons
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports
+
+  docs-build-deploy:
+    docker:
+      - image: circleci/python:3.7-node
+    steps:
+      - checkout
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-python3.7-{{ checksum "requirements.txt" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-python3.7-
+      - run:
+          name: Install dependencies
+          # Note that we the circleci node image installs stuff with a user "circleci", rather
+          # than root. So we need to tell npm where to install stuff.
+          command: |
+            npm set prefix=/home/circleci/npm
+            npm install -g --silent [email protected]
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install -r requirements.txt
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-python3.7-{{ checksum "requirements.txt" }}
+      - add_ssh_keys:
+          # This SSH key is the "CircleCI write key v2" in https://github.com/move-coop/parsons/settings/keys
+          # We need write access to the Parsons repo, so we can push the "gh-pages" branch.
+          fingerprints:
+            - "45:3d:91:e1:ae:bd:2c:b6:d4:7c:4f:b1:20:6c:1f:6c"
+      - run:
+          name: Build and deploy docs
+          # When running gh-pages, we specify to include dotfiles, so we pick up the .nojerkyll file.
+          # (This file tell Github Pages that we want to include all files in docs/, including those
+          # that start with an underscore like _static/).
+          command: |
+            . venv/bin/activate
+            cd docs/ && make html && cd ..
+            git config user.email "[email protected]"
+            git config user.name "ci-build"
+            export PATH=/home/circleci/npm/bin:$PATH
+            gh-pages --dotfiles --dist docs
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - test
+      - docs-build-deploy:
+          filters:
+            branches:
+              only: master
+
+

.dockerignore

@@ -0,0 +1,18 @@
+.dockerignore
+Dockerfile
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+env
+pip-log.txt
+pip-delete-this-directory.txt
+.tox
+.coverage
+.coverage.*
+.cache
+coverage.xml
+*,cover
+*.log
+.git

.gitignore

@@ -0,0 +1,114 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+#docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+*.ipynb
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# scratch
+scratch*
+old!_*
+
+doctrees/
+.buildinfo
+
+.DS_Store

Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.7
+
+####################
+## Selenium setup ##
+####################
+
+# TODO Remove when we have a TMC-specific Docker image
+
+# Much of this was pulled from examples at https://github.com/joyzoursky/docker-python-chromedriver
+
+# install google chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
+RUN apt-get -y update
+RUN apt-get install -y google-chrome-stable
+
+# install chromedriver
+RUN apt-get install -yqq unzip
+RUN wget -O /tmp/chromedriver.zip http://chromedriver.storage.googleapis.com/`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip
+RUN unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/
+
+# set display port to avoid crash
+ENV DISPLAY=:99
+
+###################
+## Parsons setup ##
+###################
+
+RUN mkdir /src
+
+COPY requirements.txt /src/
+RUN pip install -r /src/requirements.txt
+
+COPY . /src/
+WORKDIR /src
+
+RUN python setup.py develop
+
+# The /app directory can house the scripts that will actually execute on this Docker image.
+# Eg. If using this image in a Civis container script, Civis will install your script repo
+# (from Github) to /app.
+RUN mkdir /app
+WORKDIR /app
+# Useful for importing modules that are associated with your python scripts:
+env PYTHONPATH=.:/app

LICENSE.txt

@@ -0,0 +1 @@
+DROP IN HERE...

README.md

@@ -0,0 +1,75 @@
+# Parsons
+
+
+## About
+Parsons, named after [Lucy Parsons](https://en.wikipedia.org/wiki/Lucy_Parsons), is a Python package that contains a growing list of connectors and integrations to move data between various tools. Parsons is focused on integrations and connectors for tools utilized by the progressive community.
+
+Parsons is maintained by The Movement Cooperative. The Movement Cooperative is a member led organization focused on providing data, tools and strategic support for the progressive community.
+
+Parsons is only compatible with Python 3.x
+
+## Installation
+
+Download this repository and then run `python setup.py develop`.
+
+## Documentation
+Our package documentation is hosted at [https://move-coop.github.io/parsons/html/index.html](https://move-coop.github.io/parsons/html/index.html). It is intended for developers _using_ Parsons.
+
+The docs files are automatically built and deployed to that website as part of our CircleCI config.yml.
+
+If you want to build the docs manually (eg. to test your updates locally):
+- `cd docs/`
+- `make html`
+    - **Note**: This only creates html for files that have been modified. To create html for all files add the `-a` flag to `SPHINXOPTS    =` in Makefile.
+- Open `docs/index.html` in your web browser.
+
+Documentation for developers _working on_ Parsons should live in this README, rather than the hosted documentation website.
+
+## Tests
+Run all tests with this command:
+
+``pytest -rf test/``
+
+Run all tests in a file with:
+
+``pytest -rf test/[test_some_module.py]``
+
+Run test(s) with name(s) matching a string:
+
+``pytest -rf -k [test_something]``
+
+If you want to see all the console output for your tests (even those that pass), add the ``-s`` flag.
+
+See the [pytest documentation](https://docs.pytest.org/en/latest/contents.html) for more info and many more options.
+
+Many Parsons tests are built to hit a live server and will not run unless you set the environmental variable ``LIVE_TEST='TRUE'``
+
+**Note**: We are in process of moving from python's native unittest system over to pytest. So you'll see tests written in both styles. The above pytest command will pull in both kinds of tests.
+
+To "lint" the codebase to ensure it matches our preferred style:
+
+``flake8 --max-line-length=100 parsons``
+
+## Logging
+
+Parsons uses the [native python logging system](https://docs.python.org/3/howto/logging.html). By default, log output will go to the console and look like:
+
+```
+parsons.modulename LOGLEVEL the specific log message
+```
+
+If you're writing a Parsons module, set up your logger at the top of the file, like this:
+
+```python
+import logging
+logger = logging.getLogger(__name__)
+
+# Then to use the logger:
+# logger.info("processing a table")
+```
+
+## Best practices
+
+### Table vs native python collection
+
+When writing Parsons code, you'll often be faced with a choice of whether to store data in a Parsons Table or a native python collection (eg. list or dict). The general rule of thumb is use a Table when dealing with data that will make use of the ETL functionality Tables provide. Ie. If your data can be arbitrarily large, and/or if it has multiple fields per item (2D data), it is a good candidate for going into a Table. If your data is small and simple, eg. a list of files, we prefer simple python types like lists.

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    = -a
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Parsons
+SOURCEDIR     = .
+BUILDDIR      = .
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)