rewrite makefile: pluto, dobjobs

moving scripts to makefile, currently working for pluto & dobjobs

Makefile

@@ -1,11 +1,62 @@
+#------------------------#
+# NYC-DB                 #
+#------------------------#
+
+# CONNECTION VARIABLES
+DB_HOST='127.0.0.1'
+DB_DATABASE=nycdb
+DB_USER=nycdb
+DB_PASSWORD=nycdb
+
+# exporting allows these variables
+# to be accessed in the subshells
+# required for the template.sh to work
+export DB_HOST
+export DB_DATABASE
+export DB_USER
+export DB_PASSWORD
+
+NYCDB_DOCKER=
+
+# use BASH as our sell
+SHELL=/bin/bash
+
 default: help
 
+nyc-db: prepare-docker pluto dobjobs
+
 download:
 	./scripts/download.sh all
 
 download-pluto-all:
 	./scripts/download.sh all --pluto-all
 
+.PHONY: pluto
+pluto:
+	./scripts/template.sh > ./modules/pluto/pg_setup.sh
+	echo "pluto_root=$(shell pwd)/data/pluto/" >> modules/pluto/pg_setup.sh
+	cd modules/pluto && make && ./pluto16v2.sh
+
+JOB_FILINGS_PATH=$(shell pwd)/data/dobjobs/job_filings.csv
+
+.ONESHELL: dobjobs
+.PHONY : dobjobs
+dobjobs:
+	@echo "Inserting DOB data into postgres"
+	set -eu
+	./scripts/template.sh > ./modules/dobjobs/env.sh
+	cd modules/dobjobs
+	make install
+	./venv/bin/dobjobs  --psql -H $(DB_HOST) -U $(DB_USER) -P $(DB_PASSWORD) -D $(DB_DATABASE) "$(JOB_FILINGS_PATH)"
+	@echo "Indexing and Processing DOB Data"
+	source env.sh
+	execute_sql sql/geocode.sql
+	execute_sql sql/add_columns.sql
+	execute_sql sql/index.sql
+	rm env.sh
+
+
+.PHONY : docker-setup
 docker-setup:
 	mkdir -p postgres-data
 	docker pull aepyornis/nyc-db:0.0.1
@@ -15,7 +66,7 @@ docker-download:
 	docker-compose run nycdb bash -c "cd /opt/nyc-db && make download"
 
 docker-run:
-	docker-compose run nycdb bash -c "NYCDB_DOCKER=true cd /opt/nyc-db && make nyc-db"
+	docker-compose run nycdb bash -c "cd /opt/nyc-db && make nyc-db NYCDB_DOCKER=true DB_DATABASE=postgres DB_USER=postgres DB_HOST=pg"
 
 docker-shell:
 	PGPASSWORD=nycdb psql -U postgres -h 127.0.0.1
@@ -26,14 +77,20 @@ docker-db-standalone:
 docker-dump:
 	docker-compose run pg pg_dump --no-owner --clean --if-exists -h pg -U postgres --file=/opt/nyc-db/nyc-db.sql postgres 
 
-nyc-db:
-	./scripts/nyc_db.sh
+
+preparse-docker: 
+ifdef NYCDB_DOCKER
+	@echo 'Running as docker!'
+	./scripts/docker_setup.sh
+else
+	@echo '-'
+endif
 
 nyc-db-pluto-all:
 	./scripts/nyc_db.sh --pluto-all
 
-.ONESHELL:
-SHELL=/bin/bash
+
+.ONESHELL: db-dump
 db-dump:
 	source ./env.sh
 	pg_dump --no-owner --clean --if-exists -h 127.0.0.1 -U nycdb nycdb > "nyc-db-$$(date +%F).sql"
@@ -44,7 +101,9 @@ db-dump-bzip:
 remove-venv:
 	rm -rf modules/dof-sales/venv
 	rm -rf modules/pluto/venv
+	rm -rf modules/dobjobs/venv
 
+.PHONY : clean
 clean: remove-venv
 	rm -rf postgres-data
 	docker-compose rm -f

scripts/dobjobs.sh

@@ -6,10 +6,13 @@ source ./env.sh
 pwd=$(pwd)
 mkdir -p tmp
 
+job_filings_path=${pwd}/data/dobjobs/job_filings.csv
+
 printf "Inserting DOB data into postgres\n"
 
-cd ${pwd}/modules/dobjobs/csvparser
-job_filings_path=${pwd}/data/dobjobs/job_filings.csv
+cd ${pwd}/modules/dobjobs
+make install
+
 python3 db_dobjobs.py $job_filings_path "${NYCDB_CONNECTION_STRING}"
 
 printf "Indexing and Processing DOB Data\n"

scripts/docker_setup.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [ -f ./env.sh ];then
+    mv ./env.sh ./env.sh.backup
+fi
+
+cp docker_env.sh env.sh
+
+if [ ! -e ./env.sh ];then
+    printf "MISSING env.sh\n"
+    exit 1
+fi
+
+source ./env.sh