From 74f0cee3323640854a510becdd172fbde8b87c84 Mon Sep 17 00:00:00 2001 From: austinhallock Date: Thu, 11 Jun 2020 16:48:26 -0500 Subject: [PATCH] js fixes --- Dockerfile | 2 +- Gulpfile.mjs | 13 +- backend-shared | 2 +- bin/server.js | 21 +- bulk-decaffeinate.config.js | 9 - config.js | 2 +- graphql/directives.js | 7 +- graphql/irs_person/model.js | 24 +- graphql/irs_person/resolvers.js | 5 +- index.js | 114 ++-- package-lock.json | 19 +- package.json | 3 +- services/cache.js | 2 +- services/irs_990_importer/format_irs_990.js | 230 ++++---- services/irs_990_importer/format_irs_990ez.js | 201 ++++--- services/irs_990_importer/format_irs_990pf.js | 496 +++++++++--------- services/irs_990_importer/helpers.js | 94 ++-- services/irs_990_importer/index.js | 209 ++++---- .../irs_990_importer/load_all_for_year.js | 110 ++-- services/irs_990_importer/ntee.js | 98 ++-- services/irs_990_importer/parse_websites.js | 2 +- services/irs_990_importer/set_ntee.js | 80 ++- services/setup.js | 44 +- 23 files changed, 870 insertions(+), 917 deletions(-) delete mode 100644 bulk-decaffeinate.config.js diff --git a/Dockerfile b/Dockerfile index 3dcac14..e564865 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12.16.1-buster +FROM node:14.4.0 RUN apt-get update && apt-get install -y gdal-bin git python python-pip graphicsmagick imagemagick libcairo2-dev libjpeg-dev libpango1.0-dev libgif-dev librsvg2-dev build-essential diff --git a/Gulpfile.mjs b/Gulpfile.mjs index 9a93db5..c4adbf1 100644 --- a/Gulpfile.mjs +++ b/Gulpfile.mjs @@ -1,5 +1,3 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. import gulp from 'gulp' import { spawn } from 'child_process' @@ -11,21 +9,16 @@ const paths = { ] } -gulp.task('default', 'dev') - -gulp.task('dev', 'watch:dev') - -gulp.task('watch:dev', gulp.series('dev:server', () => gulp.watch(paths.js, ['dev:server'])) -) - let devServer = null gulp.task('dev:server', function () { process.on('exit', () => devServer?.kill()) devServer && devServer.kill() - devServer = spawn('js', [paths.serverBin], { stdio: 'inherit' }) + devServer = spawn('node', [paths.serverBin], { stdio: 'inherit' }) return devServer.on('close', function (code) { if (code === 8) { return gulp.log('Error detected, waiting for changes') } }) }) + +gulp.task('dev', gulp.parallel('dev:server', () => gulp.watch(paths.js, gulp.series('dev:server')))) diff --git a/backend-shared b/backend-shared index acf3f14..cfdcbdf 160000 --- a/backend-shared +++ b/backend-shared @@ -1 +1 @@ -Subproject commit acf3f14ae7b80f4eff1f5b697b7ced007ea63ba0 +Subproject commit cfdcbdfadd2dba9fdabe6d3307eb2a346816acaa diff --git a/bin/server.js b/bin/server.js index 8a2900b..57594f9 100644 --- a/bin/server.js +++ b/bin/server.js @@ -1,9 +1,8 @@ -import log from 'loga' import cluster from 'cluster' import os from 'os' import _ from 'lodash' -import { setup, childSetup, server } from '../index.js' +import { setup, childSetup, serverPromise } from '../index.js' import config from '../config.js' if (config.ENV === config.ENVS.PROD) { @@ -17,22 +16,24 @@ if (config.ENV === config.ENVS.PROD) { }) return cluster.on('exit', function (worker) { - log(`Worker ${worker.id} died, respawning`) + console.log(`Worker ${worker.id} died, respawning`) return cluster.fork() }) - }).catch(log.error) + }).catch(console.log) } else { - childSetup().then(() => + childSetup().then(async () => { + const server = await serverPromise server.listen(config.PORT, () => - log.info('Worker %d, listening on %d', cluster.worker.id, config.PORT) + console.log('Worker %d, listening on %d', cluster.worker.id, config.PORT) ) - ) + }) } } else { console.log('Setting up') - setup().then(() => + setup().then(async () => { + const server = await serverPromise server.listen(config.PORT, () => - log.info('Server listening on port %d', config.PORT) + console.log('Server listening on port %d', config.PORT) ) - ).catch(log.error) + }).catch(console.log) } diff --git a/bulk-decaffeinate.config.js b/bulk-decaffeinate.config.js deleted file mode 100644 index f8026a0..0000000 --- a/bulk-decaffeinate.config.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = { - decaffeinatePath: '../../misc_git/decaffeinate/bin/decaffeinate', - jscodeshiftScripts: ['prefer-function-declarations.js'], - // searchDirectory: 'src', - decaffeinateArgs: [ - '--use-js-modules', '--use-cs2', '--loose', '--disable-suggestion-comment', - '--use-optional-chaining' - ] -} diff --git a/config.js b/config.js index 7eed3f0..077b4ae 100644 --- a/config.js +++ b/config.js @@ -4,7 +4,7 @@ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. import _ from 'lodash' -import assertNoneMissing from 'assert-none-missing' +import { assertNoneMissing } from 'backend-shared' const { env diff --git a/graphql/directives.js b/graphql/directives.js index 3a90ab7..3dfb6d7 100644 --- a/graphql/directives.js +++ b/graphql/directives.js @@ -1,6 +1,9 @@ import { Format } from 'backend-shared' -import { defaultFieldResolver } from 'graphql' -import { SchemaDirectiveVisitor } from 'graphql-tools' +import GraphQL from 'graphql' +import GraphQLTools from 'graphql-tools' + +const { defaultFieldResolver } = GraphQL +const { SchemaDirectiveVisitor } = GraphQLTools export const nameCase = class NameCase extends SchemaDirectiveVisitor { visitFieldDefinition (field) { diff --git a/graphql/irs_person/model.js b/graphql/irs_person/model.js index 537b7bf..03d3267 100644 --- a/graphql/irs_person/model.js +++ b/graphql/irs_person/model.js @@ -1,29 +1,7 @@ -/* eslint-disable - constructor-super, - no-constant-condition, - no-eval, - no-this-before-super, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. import _ from 'lodash' -import { Base, cknex, elasticsearch } from 'backend-shared' -import config from '../../config' +import { Base, cknex } from 'backend-shared' class IrsPersonModel extends Base { - constructor (...args) { - { - // Hack: trick Babel/TypeScript into allowing this before super. - if (false) { super() } - const thisFn = (() => { return this }).toString() - const thisName = thisFn.match(/return (?:_assertThisInitialized\()*(\w+)\)*;/)[1] - eval(`${thisName} = this;`) - } - this.getAllByEin = this.getAllByEin.bind(this) - super(...args) - } - getScyllaTables () { return [ { diff --git a/graphql/irs_person/resolvers.js b/graphql/irs_person/resolvers.js index 96ed472..c03c0fc 100644 --- a/graphql/irs_person/resolvers.js +++ b/graphql/irs_person/resolvers.js @@ -1,7 +1,6 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. import { GraphqlFormatter } from 'backend-shared' -import IrsPerson from './model' + +import IrsPerson from './model.js' export const Query = { irsPersons (rootValue, { ein, query, limit }) { diff --git a/index.js b/index.js index 03ae83c..ec024bb 100644 --- a/index.js +++ b/index.js @@ -1,34 +1,36 @@ import fs from 'fs' -import _ from 'lodash' import cors from 'cors' import express from 'express' import Promise from 'bluebird' import bodyParser from 'body-parser' import http from 'http' -import { ApolloServer } from 'apollo-server-express' -import { buildFederatedSchema } from '@apollo/federation' -import { SchemaDirectiveVisitor } from 'graphql-tools' +import ApolloServerExpress from 'apollo-server-express' +import ApolloFederation from '@apollo/federation' +import GraphQLTools from 'graphql-tools' +import { dirname } from 'path' +import { fileURLToPath } from 'url' import { Schema, elasticsearch } from 'backend-shared' -import helperConfig from 'backend-shared/lib/config.js' import { setup, childSetup } from './services/setup.js' import { setNtee } from './services/irs_990_importer/set_ntee.js' import { loadAllForYear } from './services/irs_990_importer/load_all_for_year.js' import { processUnprocessedOrgs, processEin, fixBadFundImports, processUnprocessedFunds -} from './services/irs_990_importer.js' +} from './services/irs_990_importer/index.js' import { parseGrantMakingWebsites } from './services/irs_990_importer/parse_websites.js' import IrsOrg990 from './graphql/irs_org_990/model.js' -import directives from './graphql/directives.js.js' +import * as directives from './graphql/directives.js' import config from './config.js' -// FIXME: change to a setup fn that everything else waits on -helperConfig.set(_.pick(config, config.SHARED_WITH_PHIL_HELPERS)) +const { ApolloServer } = ApolloServerExpress +const { buildFederatedSchema } = ApolloFederation +const { SchemaDirectiveVisitor } = GraphQLTools +const __dirname = dirname(fileURLToPath(import.meta.url)) let resolvers, schemaDirectives let typeDefs = fs.readFileSync('./graphql/type.graphql', 'utf8') -let schema = Schema.getSchema({ directives, typeDefs, dirName: __dirname }) +const schemaPromise = Schema.getSchema({ directives, typeDefs, dirName: __dirname }) Promise.config({ warnings: false }) @@ -50,7 +52,7 @@ app.get('/tableCount', function (req, res) { if (validTables.indexOf(req.query.tableName) === -1) { res.send({ error: 'invalid table name' }) } - return elasticsearch.count({ + return elasticsearch.client.count({ index: req.query.tableName }) .then(c => res.send(JSON.stringify(c))) @@ -89,14 +91,14 @@ app.get('/setES', async function (req, res) { // refreshInterval = null return res.send(await Promise.map(validTables, async function (tableName) { - const settings = await elasticsearch.indices.getSettings({ + const settings = await elasticsearch.client.indices.getSettings({ index: tableName }) const previous = settings[tableName].settings.index const diff = { number_of_replicas: replicas } // refresh_interval: refreshInterval - await elasticsearch.indices.putSettings({ + await elasticsearch.client.indices.putSettings({ index: tableName, body: diff }) @@ -115,7 +117,7 @@ app.get('/setMaxWindow', async function (req, res) { res.send({ error: 'must be number between 10,000 and 100,000' }) } - return res.send(await elasticsearch.indices.putSettings({ + return res.send(await elasticsearch.client.indices.putSettings({ index: req.query.tableName, body: { max_result_window: maxResultWindow } })) @@ -183,50 +185,52 @@ app.get('/processUnprocessedFunds', function (req, res) { app.get('/parseGrantMakingWebsites', function (req, res) { parseGrantMakingWebsites() return res.send('syncing') -}); - -({ typeDefs, resolvers, schemaDirectives } = schema) -schema = buildFederatedSchema({ typeDefs, resolvers }) -// https://github.com/apollographql/apollo-feature-requests/issues/145 -SchemaDirectiveVisitor.visitSchemaDirectives(schema, schemaDirectives) - -const defaultQuery = ` -query($query: ESQuery!) { - irsOrgs(query: $query) { - nodes { - name - employeeCount - volunteerCount +}) + +const serverPromise = schemaPromise.then((schema) => { + ({ typeDefs, resolvers, schemaDirectives } = schema) + schema = buildFederatedSchema({ typeDefs, resolvers }) + // https://github.com/apollographql/apollo-feature-requests/issues/145 + SchemaDirectiveVisitor.visitSchemaDirectives(schema, schemaDirectives) + + const defaultQuery = ` + query($query: ESQuery!) { + irsOrgs(query: $query) { + nodes { + name + employeeCount + volunteerCount + } } } -} -` - -const defaultQueryVariables = ` -{ - "query": {"range": {"volunteerCount": {"gte": 10000}}} -} -` - -const graphqlServer = new ApolloServer({ - schema, - introspection: true, - playground: { - // settings: - tabs: [ - { - endpoint: config.ENV === config.ENVS.DEV - ? `http://localhost:${config.PORT}/graphql` - : 'https://api.techby.org/990/v1/graphql', - query: defaultQuery, - variables: defaultQueryVariables - } - ] + ` + + const defaultQueryVariables = ` + { + "query": {"range": {"volunteerCount": {"gte": 10000}}} } + ` + + const graphqlServer = new ApolloServer({ + schema, + introspection: true, + playground: { + // settings: + tabs: [ + { + endpoint: config.ENV === config.ENVS.DEV + ? `http://localhost:${config.PORT}/graphql` + : 'https://api.techby.org/990/v1/graphql', + query: defaultQuery, + variables: defaultQueryVariables + } + ] + } -}) -graphqlServer.applyMiddleware({ app, path: '/graphql' }) + }) + graphqlServer.applyMiddleware({ app, path: '/graphql' }) -const server = http.createServer(app) + return http.createServer(app) +}) -export { server, setup, childSetup } +export { serverPromise, setup, childSetup } diff --git a/package-lock.json b/package-lock.json index 3de3552..ad09490 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1358,18 +1358,10 @@ } }, "assert-none-missing": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/assert-none-missing/-/assert-none-missing-0.1.2.tgz", - "integrity": "sha1-ZYHjm5rjM7oaFbWzle0FgA9znn4=", + "version": "github:claydotio/assert-none-missing#716303db0e5bf51058c5e048652eb5ca701468a9", + "from": "github:claydotio/assert-none-missing#es", "requires": { - "lodash": "^3.10.0" - }, - "dependencies": { - "lodash": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", - "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" - } + "lodash-es": "^4.17.15" } }, "assert-plus": { @@ -5114,6 +5106,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, + "lodash-es": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.15.tgz", + "integrity": "sha512-rlrc3yU3+JNOpZ9zj5pQtxnx2THmvRykwL4Xlxoa8I9lHBlVbbyPhgyPMioxVZ4NqyxaVVtaJnzsyOidQIhyyQ==" + }, "lodash._basecopy": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz", diff --git a/package.json b/package.json index 7646f2a..fe70d46 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "private": true, "description": "", "main": "index.js", + "type": "module", "directories": { "test": "test" }, @@ -28,7 +29,7 @@ "dependencies": { "@apollo/federation": "^0.16.0", "apollo-server-express": "^2.14.2", - "assert-none-missing": "^0.1.2", + "assert-none-missing": "github:claydotio/assert-none-missing#es", "backend-shared": "file:backend-shared", "bluebird": "^3.5.3", "body-parser": "^1.14.1", diff --git a/services/cache.js b/services/cache.js index c02dedd..a48cbaa 100644 --- a/services/cache.js +++ b/services/cache.js @@ -1,4 +1,4 @@ -export const PREFIXES = { +export default { EIN_FROM_NAME: 'ein:name', ENTITY_ID: 'entity:id2', ENTITY_SLUG: 'entity:slug1' diff --git a/services/irs_990_importer/format_irs_990.js b/services/irs_990_importer/format_irs_990.js index 9aeaa87..c22a6f7 100644 --- a/services/irs_990_importer/format_irs_990.js +++ b/services/irs_990_importer/format_irs_990.js @@ -2,133 +2,129 @@ import _ from 'lodash' import { formatInt, formatBigInt, formatWebsite, formatFloat, getOrgNameByFiling } from './helpers.js' -export default { - getOrg990Json (filing, { ein, year }) { - const entityName = getOrgNameByFiling(filing) +export function getOrg990Json (filing, { ein, year }) { + const entityName = getOrgNameByFiling(filing) - const exemptStatus = (() => { - if (filing.IRS990.parts.part_0?.Orgnztn527Ind) { - return '527' - } else if (filing.IRS990.parts.part_0?.Orgnztn49471NtPFInd) { - return '4947a1' - } else if (filing.IRS990.parts.part_0?.Orgnztn501c3Ind) { - return '501c3' - } - })() - // https://github.com/jsfenfen/990-xml-reader/issues/26 - // else if filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd - // then "501c#{filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd}" + let exemptStatus + if (filing.IRS990.parts.part_0?.Orgnztn527Ind) { + exemptStatus = '527' + } else if (filing.IRS990.parts.part_0?.Orgnztn49471NtPFInd) { + exemptStatus = '4947a1' + } else if (filing.IRS990.parts.part_0?.Orgnztn501c3Ind) { + exemptStatus = '501c3' + } + // https://github.com/jsfenfen/990-xml-reader/issues/26 + // else if filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd + // then "501c#{filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd}" - return { - ein, - year, - name: entityName, - city: filing.ReturnHeader.USAddrss_CtyNm, - state: filing.ReturnHeader.USAddrss_SttAbbrvtnCd, - website: formatWebsite(filing.IRS990.parts.part_0?.WbstAddrssTxt), - exemptStatus, - mission: filing.IRS990.parts.part_i?.ActvtyOrMssnDsc, - revenue: _.pickBy({ - investments: formatBigInt(filing.IRS990.parts.part_i?.CYInvstmntIncmAmt), - grants: formatBigInt(filing.IRS990.parts.part_i?.CYGrntsAndSmlrPdAmt), - ubi: formatBigInt(filing.IRS990.parts.part_i?.TtlGrssUBIAmt), // ** - netUbi: formatBigInt(filing.IRS990.parts.part_i?.NtUnrltdBsTxblIncmAmt), - contributionsAndGrants: formatBigInt(filing.IRS990.parts.part_i?.CYCntrbtnsGrntsAmt), - programService: formatBigInt(filing.IRS990.parts.part_i?.CYPrgrmSrvcRvnAmt), - other: formatBigInt(filing.IRS990.parts.part_i?.CYOthrRvnAmt), - total: formatBigInt(filing.IRS990.parts.part_i?.CYTtlRvnAmt) - }), + return { + ein, + year, + name: entityName, + city: filing.ReturnHeader.USAddrss_CtyNm, + state: filing.ReturnHeader.USAddrss_SttAbbrvtnCd, + website: formatWebsite(filing.IRS990.parts.part_0?.WbstAddrssTxt), + exemptStatus, + mission: filing.IRS990.parts.part_i?.ActvtyOrMssnDsc, + revenue: _.pickBy({ + investments: formatBigInt(filing.IRS990.parts.part_i?.CYInvstmntIncmAmt), + grants: formatBigInt(filing.IRS990.parts.part_i?.CYGrntsAndSmlrPdAmt), + ubi: formatBigInt(filing.IRS990.parts.part_i?.TtlGrssUBIAmt), // ** + netUbi: formatBigInt(filing.IRS990.parts.part_i?.NtUnrltdBsTxblIncmAmt), + contributionsAndGrants: formatBigInt(filing.IRS990.parts.part_i?.CYCntrbtnsGrntsAmt), + programService: formatBigInt(filing.IRS990.parts.part_i?.CYPrgrmSrvcRvnAmt), + other: formatBigInt(filing.IRS990.parts.part_i?.CYOthrRvnAmt), + total: formatBigInt(filing.IRS990.parts.part_i?.CYTtlRvnAmt) + }), - paidBenefitsToMembers: formatBigInt(filing.IRS990.parts.part_i?.CYBnftsPdTMmbrsAmt), - expenses: _.pickBy({ - salaries: formatBigInt(filing.IRS990.parts.part_i?.CYSlrsCmpEmpBnftPdAmt), - professionalFundraising: formatBigInt(filing.IRS990.parts.part_i?.CYTtlPrfFndrsngExpnsAmt), - fundraising: formatBigInt(filing.IRS990.parts.part_i?.CYTtlPrfFndrsngExpnsAmt), - other: formatBigInt(filing.IRS990.parts.part_i?.CYOthrExpnssAmt), - total: formatBigInt(filing.IRS990.parts.part_i?.CYTtlExpnssAmt) - }), // ** - assets: _.pickBy({ - boy: formatBigInt(filing.IRS990.parts.part_i?.TtlAsstsBOYAmt), - eoy: formatBigInt(filing.IRS990.parts.part_i?.TtlAsstsEOYAmt) - }), - liabilities: _.pickBy({ - boy: formatBigInt(filing.IRS990.parts.part_i?.TtlLbltsBOYAmt), - eoy: formatBigInt(filing.IRS990.parts.part_i?.TtlLbltsEOYAmt) - }), - netAssets: _.pickBy({ - boy: formatBigInt(filing.IRS990.parts.part_i?.NtAsstsOrFndBlncsBOYAmt), - eoy: formatBigInt(filing.IRS990.parts.part_i?.NtAsstsOrFndBlncsEOYAmt) - }), // ** + paidBenefitsToMembers: formatBigInt(filing.IRS990.parts.part_i?.CYBnftsPdTMmbrsAmt), + expenses: _.pickBy({ + salaries: formatBigInt(filing.IRS990.parts.part_i?.CYSlrsCmpEmpBnftPdAmt), + professionalFundraising: formatBigInt(filing.IRS990.parts.part_i?.CYTtlPrfFndrsngExpnsAmt), + fundraising: formatBigInt(filing.IRS990.parts.part_i?.CYTtlPrfFndrsngExpnsAmt), + other: formatBigInt(filing.IRS990.parts.part_i?.CYOthrExpnssAmt), + total: formatBigInt(filing.IRS990.parts.part_i?.CYTtlExpnssAmt) + }), // ** + assets: _.pickBy({ + boy: formatBigInt(filing.IRS990.parts.part_i?.TtlAsstsBOYAmt), + eoy: formatBigInt(filing.IRS990.parts.part_i?.TtlAsstsEOYAmt) + }), + liabilities: _.pickBy({ + boy: formatBigInt(filing.IRS990.parts.part_i?.TtlLbltsBOYAmt), + eoy: formatBigInt(filing.IRS990.parts.part_i?.TtlLbltsEOYAmt) + }), + netAssets: _.pickBy({ + boy: formatBigInt(filing.IRS990.parts.part_i?.NtAsstsOrFndBlncsBOYAmt), + eoy: formatBigInt(filing.IRS990.parts.part_i?.NtAsstsOrFndBlncsEOYAmt) + }), // ** - votingMemberCount: formatInt(filing.IRS990.parts.part_i?.VtngMmbrsGvrnngBdyCnt), - independentVotingMemberCount: formatInt(filing.IRS990.parts.part_i?.VtngMmbrsIndpndntCnt), + votingMemberCount: formatInt(filing.IRS990.parts.part_i?.VtngMmbrsGvrnngBdyCnt), + independentVotingMemberCount: formatInt(filing.IRS990.parts.part_i?.VtngMmbrsIndpndntCnt), - employeeCount: formatInt(filing.IRS990.parts.part_i?.TtlEmplyCnt), // ** - volunteerCount: formatInt(filing.IRS990.parts.part_i?.TtlVlntrsCnt) // ** - } - }, + employeeCount: formatInt(filing.IRS990.parts.part_i?.TtlEmplyCnt), // ** + volunteerCount: formatInt(filing.IRS990.parts.part_i?.TtlVlntrsCnt) // ** + } +} - // 990ez / 990pf - getOrgJson (org990, persons, existing990s) { - const org = { - // TODO: org type (501..) - ein: org990.ein, - name: org990.name, - city: org990.city, - state: org990.state, - website: org990.website, - mission: org990.mission, - exemptStatus: org990.exemptStatus - } +// 990ez / 990pf +export function getOrgJson (org990, persons, existing990s) { + const org = { + // TODO: org type (501..) + ein: org990.ein, + name: org990.name, + city: org990.city, + state: org990.state, + website: org990.website, + mission: org990.mission, + exemptStatus: org990.exemptStatus + } - const maxExistingYear = _.maxBy(existing990s, 'year')?.year - if ((org990.year >= maxExistingYear) || !maxExistingYear) { - org.maxYear = org990.year - org.assets = org990.assets.eoy - org.netAssets = org990.netAssets.eoy - org.liabilities = org990.liabilities.eoy - org.employeeCount = org990.employeeCount - org.volunteerCount = org990.volunteerCount + const maxExistingYear = _.maxBy(existing990s, 'year')?.year + if ((org990.year >= maxExistingYear) || !maxExistingYear) { + org.maxYear = org990.year + org.assets = org990.assets.eoy + org.netAssets = org990.netAssets.eoy + org.liabilities = org990.liabilities.eoy + org.employeeCount = org990.employeeCount + org.volunteerCount = org990.volunteerCount - org.lastYearStats = { - year: org990.year, - revenue: org990.revenue.total, - expenses: org990.expenses.total, - topSalary: _.pick(_.maxBy(persons, 'compensation'), [ - 'name', 'title', 'compensation' - ]) - } + org.lastYearStats = { + year: org990.year, + revenue: org990.revenue.total, + expenses: org990.expenses.total, + topSalary: _.pick(_.maxBy(persons, 'compensation'), [ + 'name', 'title', 'compensation' + ]) } + } - return org - }, - - // TODO: mark people from previous years as inactive people for org - getOrgPersonsJson (filing) { - const entityName = getOrgNameByFiling(filing) + return org +} - return _.map(filing.IRS990.groups.Frm990PrtVIISctnA, function (person) { - let businessName = person.BsnssNmLn1Txt - if (person.BsnssNmLn2Txt) { - businessName += ` ${person.BsnssNmLn2Txt}` - } - return { - name: person.PrsnNm || businessName, - entityName, - entityType: 'org', - year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), - isBusiness: Boolean(businessName), - title: person.TtlTxt, - weeklyHours: formatFloat(person.AvrgHrsPrWkRt || person.AvrgHrsPrWkRltdOrgRt), - compensation: formatInt(person.RprtblCmpFrmOrgAmt), - relatedCompensation: formatInt(person.RprtblCmpFrmRltdOrgAmt), - otherCompensation: formatInt(person.OthrCmpnstnAmt), - isOfficer: person.OffcrInd === 'X', - isFormerOfficer: person.FrmrOfcrDrctrTrstInd === 'X', - isKeyEmployee: person.KyEmplyInd === 'X', - isHighestPaidEmployee: person.HghstCmpnstdEmplyInd === 'X' - } - }) - } +// TODO: mark people from previous years as inactive people for org +export function getOrgPersonsJson (filing) { + const entityName = getOrgNameByFiling(filing) + return _.map(filing.IRS990.groups.Frm990PrtVIISctnA, function (person) { + let businessName = person.BsnssNmLn1Txt + if (person.BsnssNmLn2Txt) { + businessName += ` ${person.BsnssNmLn2Txt}` + } + return { + name: person.PrsnNm || businessName, + entityName, + entityType: 'org', + year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), + isBusiness: Boolean(businessName), + title: person.TtlTxt, + weeklyHours: formatFloat(person.AvrgHrsPrWkRt || person.AvrgHrsPrWkRltdOrgRt), + compensation: formatInt(person.RprtblCmpFrmOrgAmt), + relatedCompensation: formatInt(person.RprtblCmpFrmRltdOrgAmt), + otherCompensation: formatInt(person.OthrCmpnstnAmt), + isOfficer: person.OffcrInd === 'X', + isFormerOfficer: person.FrmrOfcrDrctrTrstInd === 'X', + isKeyEmployee: person.KyEmplyInd === 'X', + isHighestPaidEmployee: person.HghstCmpnstdEmplyInd === 'X' + } + }) } diff --git a/services/irs_990_importer/format_irs_990ez.js b/services/irs_990_importer/format_irs_990ez.js index 2dd7e15..698ef77 100644 --- a/services/irs_990_importer/format_irs_990ez.js +++ b/services/irs_990_importer/format_irs_990ez.js @@ -3,113 +3,110 @@ import _ from 'lodash' import { formatInt, formatBigInt, formatWebsite, formatFloat, getOrgNameByFiling } from './helpers.js' -export default { - getOrg990EZJson (filing, { ein, year }) { - const entityName = getOrgNameByFiling(filing) +export function getOrg990EZJson (filing, { ein, year }) { + const entityName = getOrgNameByFiling(filing) - const exemptStatus = (() => { - if (filing.IRS990EZ.parts.ez_part_0?.Orgnztn527Ind) { - return '527' - } else if (filing.IRS990EZ.parts.ez_part_0?.Orgnztn49471NtPFInd) { - return '4947a1' - } else if (filing.IRS990EZ.parts.ez_part_0?.Orgnztn501c3Ind) { - return '501c3' - } - })() - // https://github.com/jsfenfen/990-xml-reader/issues/26 - // else if filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd - // then "501c#{filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd}" - - return { - ein, - year, - name: entityName, - city: filing.ReturnHeader.USAddrss_CtyNm, - state: filing.ReturnHeader.USAddrss_SttAbbrvtnCd, - website: formatWebsite(filing.IRS990EZ.parts.ez_part_0?.WbstAddrssTxt), - exemptStatus, - mission: filing.IRS990EZ.parts.ez_part_iii?.PrmryExmptPrpsTxt, - revenue: _.pickBy({ - investments: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.InvstmntIncmAmt), - grants: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.GrntsAndSmlrAmntsPdAmt), - saleOfAssets: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.SlOfAsstsGrssAmt), // ? - saleOfInventory: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.GrssSlsOfInvntryAmt), // ? - gaming: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.GmngGrssIncmAmt), - fundraising: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.FndrsngGrssIncmAmt), - // ubi: formatBigInt filing.IRS990EZ.parts.part_i?.TtlGrssUBIAmt # ** - // netUbi: formatBigInt filing.IRS990EZ.parts.part_i?.NtUnrltdBsTxblIncmAmt - contributionsAndGrants: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.CntrbtnsGftsGrntsEtcAmt), - // member dues - programService: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.MmbrshpDsAmt), - other: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.OthrRvnTtlAmt), - total: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.TtlRvnAmt) - }), - - paidBenefitsToMembers: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.BnftsPdTOrFrMmbrsAmt), - expenses: _.pickBy({ - salaries: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.SlrsOthrCmpEmplBnftAmt), - goodsSold: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.CstOfGdsSldAmt), - sales: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.CstOrOthrBssExpnsSlAmt), - independentContractors: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.FsAndOthrPymtTIndCntrctAmt), - rent: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.OccpncyRntUtltsAndMntAmt), - printing: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.PrntngPblctnsPstgAmt), - specialEvents: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.SpclEvntsDrctExpnssAmt), - other: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.OthrExpnssTtlAmt), - total: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.TtlExpnssAmt), // ** - programServicesTotal: formatBigInt(filing.IRS990EZ.parts.ez_part_iii?.TtlPrgrmSrvcExpnssAmt) - }), - assets: _.pickBy({ - cashBoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.CshSvngsAndInvstmnts_BOYAmt), - cashEoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.CshSvngsAndInvstmnts_EOYAmt), - realEstateBoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.LndAndBldngs_BOYAmt), - realEstateEoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.LndAndBldngs_EOYAmt), - boy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.Frm990TtlAssts_BOYAmt), - eoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.Frm990TtlAssts_EOYAmt) - }), - liabilities: _.pickBy({ - boy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.SmOfTtlLblts_BOYAmt), - eoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.SmOfTtlLblts_EOYAmt) - }), - netAssets: _.pickBy({ - boy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.NtAsstsOrFndBlncs_BOYAmt), - eoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.NtAsstsOrFndBlncs_EOYAmt) - }) // ** - // - // votingMemberCount: filing.IRS990EZ.parts.part_i?.VtngMmbrsGvrnngBdyCnt - // independentVotingMemberCount: filing.IRS990EZ.parts.part_i?.VtngMmbrsIndpndntCnt - // - // employeeCount: filing.IRS990EZ.parts.part_i?.TtlEmplyCnt # ** - // volunteerCount: filing.IRS990EZ.parts.part_i?.TtlVlntrsCnt # ** + const exemptStatus = (() => { + if (filing.IRS990EZ.parts.ez_part_0?.Orgnztn527Ind) { + return '527' + } else if (filing.IRS990EZ.parts.ez_part_0?.Orgnztn49471NtPFInd) { + return '4947a1' + } else if (filing.IRS990EZ.parts.ez_part_0?.Orgnztn501c3Ind) { + return '501c3' } - }, + })() + // https://github.com/jsfenfen/990-xml-reader/issues/26 + // else if filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd + // then "501c#{filing.IRS990EZ.parts.ez_part_0?.Orgnztn501cInd}" - getOrgEZPersonsJson (filing) { - const entityName = getOrgNameByFiling(filing) + return { + ein, + year, + name: entityName, + city: filing.ReturnHeader.USAddrss_CtyNm, + state: filing.ReturnHeader.USAddrss_SttAbbrvtnCd, + website: formatWebsite(filing.IRS990EZ.parts.ez_part_0?.WbstAddrssTxt), + exemptStatus, + mission: filing.IRS990EZ.parts.ez_part_iii?.PrmryExmptPrpsTxt, + revenue: _.pickBy({ + investments: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.InvstmntIncmAmt), + grants: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.GrntsAndSmlrAmntsPdAmt), + saleOfAssets: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.SlOfAsstsGrssAmt), // ? + saleOfInventory: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.GrssSlsOfInvntryAmt), // ? + gaming: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.GmngGrssIncmAmt), + fundraising: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.FndrsngGrssIncmAmt), + // ubi: formatBigInt filing.IRS990EZ.parts.part_i?.TtlGrssUBIAmt # ** + // netUbi: formatBigInt filing.IRS990EZ.parts.part_i?.NtUnrltdBsTxblIncmAmt + contributionsAndGrants: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.CntrbtnsGftsGrntsEtcAmt), + // member dues + programService: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.MmbrshpDsAmt), + other: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.OthrRvnTtlAmt), + total: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.TtlRvnAmt) + }), - let persons = filing.IRS990EZ.groups.EZOffcrDrctrTrstEmpl - if (filing.IRS990EZ.groups.EZCmpnstnHghstPdEmpl) { - persons.concat(filing.IRS990EZ.groups.EZCmpnstnHghstPdEmpl) - } + paidBenefitsToMembers: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.BnftsPdTOrFrMmbrsAmt), + expenses: _.pickBy({ + salaries: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.SlrsOthrCmpEmplBnftAmt), + goodsSold: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.CstOfGdsSldAmt), + sales: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.CstOrOthrBssExpnsSlAmt), + independentContractors: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.FsAndOthrPymtTIndCntrctAmt), + rent: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.OccpncyRntUtltsAndMntAmt), + printing: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.PrntngPblctnsPstgAmt), + specialEvents: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.SpclEvntsDrctExpnssAmt), + other: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.OthrExpnssTtlAmt), + total: formatBigInt(filing.IRS990EZ.parts.ez_part_i?.TtlExpnssAmt), // ** + programServicesTotal: formatBigInt(filing.IRS990EZ.parts.ez_part_iii?.TtlPrgrmSrvcExpnssAmt) + }), + assets: _.pickBy({ + cashBoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.CshSvngsAndInvstmnts_BOYAmt), + cashEoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.CshSvngsAndInvstmnts_EOYAmt), + realEstateBoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.LndAndBldngs_BOYAmt), + realEstateEoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.LndAndBldngs_EOYAmt), + boy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.Frm990TtlAssts_BOYAmt), + eoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.Frm990TtlAssts_EOYAmt) + }), + liabilities: _.pickBy({ + boy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.SmOfTtlLblts_BOYAmt), + eoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.SmOfTtlLblts_EOYAmt) + }), + netAssets: _.pickBy({ + boy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.NtAsstsOrFndBlncs_BOYAmt), + eoy: formatBigInt(filing.IRS990EZ.parts.ez_part_ii?.NtAsstsOrFndBlncs_EOYAmt) + }) // ** + // + // votingMemberCount: filing.IRS990EZ.parts.part_i?.VtngMmbrsGvrnngBdyCnt + // independentVotingMemberCount: filing.IRS990EZ.parts.part_i?.VtngMmbrsIndpndntCnt + // + // employeeCount: filing.IRS990EZ.parts.part_i?.TtlEmplyCnt # ** + // volunteerCount: filing.IRS990EZ.parts.part_i?.TtlVlntrsCnt # ** + } +} + +export function getOrgEZPersonsJson (filing) { + const entityName = getOrgNameByFiling(filing) - persons = _.map(persons, function (person) { - let businessName = person.BsnssNmLn1 - if (person.BsnssNmLn2) { - businessName += ` ${person.BsnssNmLn2}` - } - return { - name: person.PrsnNm || businessName, - entityName, - entityType: 'org', - year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), - isBusiness: Boolean(businessName), - title: person.TtlTxt, - weeklyHours: formatFloat(person.AvrgHrsPrWkDvtdTPsRt || person.AvrgHrsPrWkRt), - compensation: formatInt(person.CmpnstnAmt), - expenseAccount: formatInt(person.ExpnsAccntOthrAllwncAmt), - otherCompensation: formatInt(person.EmplyBnftPrgrmAmt) - } - }) - return _.uniqBy(persons, 'name') + let persons = filing.IRS990EZ.groups.EZOffcrDrctrTrstEmpl + if (filing.IRS990EZ.groups.EZCmpnstnHghstPdEmpl) { + persons.concat(filing.IRS990EZ.groups.EZCmpnstnHghstPdEmpl) } + persons = _.map(persons, function (person) { + let businessName = person.BsnssNmLn1 + if (person.BsnssNmLn2) { + businessName += ` ${person.BsnssNmLn2}` + } + return { + name: person.PrsnNm || businessName, + entityName, + entityType: 'org', + year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), + isBusiness: Boolean(businessName), + title: person.TtlTxt, + weeklyHours: formatFloat(person.AvrgHrsPrWkDvtdTPsRt || person.AvrgHrsPrWkRt), + compensation: formatInt(person.CmpnstnAmt), + expenseAccount: formatInt(person.ExpnsAccntOthrAllwncAmt), + otherCompensation: formatInt(person.EmplyBnftPrgrmAmt) + } + }) + return _.uniqBy(persons, 'name') } diff --git a/services/irs_990_importer/format_irs_990pf.js b/services/irs_990_importer/format_irs_990pf.js index 9d28438..ddafb60 100644 --- a/services/irs_990_importer/format_irs_990pf.js +++ b/services/irs_990_importer/format_irs_990pf.js @@ -1,6 +1,4 @@ /* eslint-disable camelcase */ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. import _ from 'lodash' import Promise from 'bluebird' import md5 from 'md5' @@ -18,268 +16,266 @@ import { import { getEinNteeFromNameCityState } from './ntee.js' -export default { - getFund990Json (filing, { ein, year }) { - const entityName = getOrgNameByFiling(filing) - - const website = formatWebsite(filing.IRS990PF.parts.pf_part_viia?.SttmntsRgrdngActy_WbstAddrssTxt) - - let applicantSubmissionAddress - // us address - if (filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_ZIPCd) { - applicantSubmissionAddress = { - street1: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_AddrssLn1Txt, - street2: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_AddrssLn2Txt, - postalCode: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_ZIPCd, - city: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_CtyNm, - state: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_SttAbbrvtnCd, - countryCode: 'US' - } - // foreign address - } else if (filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_FrgnPstlCd) { - applicantSubmissionAddress = { - street1: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_AddrssLn1Txt, - street2: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_AddrssLn2Txt, - postalCode: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_FrgnPstlCd, - city: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_CtyNm, - state: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_PrvncOrSttNm, - countryCode: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_CntryCd - } - } else { - applicantSubmissionAddress = null +export function getFund990Json (filing, { ein, year }) { + const entityName = getOrgNameByFiling(filing) + + const website = formatWebsite(filing.IRS990PF.parts.pf_part_viia?.SttmntsRgrdngActy_WbstAddrssTxt) + + let applicantSubmissionAddress + // us address + if (filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_ZIPCd) { + applicantSubmissionAddress = { + street1: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_AddrssLn1Txt, + street2: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_AddrssLn2Txt, + postalCode: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_ZIPCd, + city: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_CtyNm, + state: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntUSAddrss_SttAbbrvtnCd, + countryCode: 'US' } + // foreign address + } else if (filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_FrgnPstlCd) { + applicantSubmissionAddress = { + street1: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_AddrssLn1Txt, + street2: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_AddrssLn2Txt, + postalCode: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_FrgnPstlCd, + city: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_CtyNm, + state: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_PrvncOrSttNm, + countryCode: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.RcpntFrgnAddrss_CntryCd + } + } else { + applicantSubmissionAddress = null + } - return { - ein, - year, - name: entityName, - city: filing.ReturnHeader.USAddrss_CtyNm, - state: filing.ReturnHeader.USAddrss_SttAbbrvtnCd, - website, - - revenue: _.pickBy({ - contributionsAndGrants: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CntrRcvdRvAndExpnssAmt), - interestOnSavings: formatBigInt(filing.IRS990PF.parts.pf_part_i?.IntrstOnSvRvAndExpnssAmt), - dividendsFromSecurities: formatBigInt(filing.IRS990PF.parts.pf_part_i?.DvdndsRvAndExpnssAmt), - netRental: formatBigInt(filing.IRS990PF.parts.pf_part_i?.NtRntlIncmOrLssAmt), - netAssetSales: formatBigInt(filing.IRS990PF.parts.pf_part_i?.NtGnSlAstRvAndExpnssAmt), - capitalGain: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CpGnNtIncmNtInvstIncmAmt), - capitalGainShortTerm: formatBigInt(filing.IRS990PF.parts.pf_part_i?.NtSTCptlGnAdjNtIncmAmt), - incomeModifications: formatBigInt(filing.IRS990PF.parts.pf_part_i?.IncmMdfctnsAdjNtIncmAmt), - grossSales: formatBigInt(filing.IRS990PF.parts.pf_part_i?.GrssPrftAdjNtIncmAmt), - other: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthrIncmRvAndExpnssAmt), - // ** - total: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TtlRvAndExpnssAmt) - }), - - expenses: _.pickBy({ - officerSalaries: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CmpOfcrDrTrstRvAndExpnssAmt), - nonOfficerSalaries: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthEmplSlrsWgsRvAndExpnssAmt), - employeeBenefits: formatBigInt(filing.IRS990PF.parts.pf_part_i?.PnsnEmplBnftRvAndExpnssAmt), - legalFees: formatBigInt(filing.IRS990PF.parts.pf_part_i?.LglFsRvAndExpnssAmt), - accountingFees: formatBigInt(filing.IRS990PF.parts.pf_part_i?.AccntngFsRvAndExpnssAmt), - otherProfessionalFees: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthrPrfFsRvAndExpnssAmt), - interest: formatBigInt(filing.IRS990PF.parts.pf_part_i?.IntrstRvAndExpnssAmt), - taxes: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TxsRvAndExpnssAmt), - depreciation: formatBigInt(filing.IRS990PF.parts.pf_part_i?.DprcAndDpltnRvAndExpnssAmt), - occupancy: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OccpncyRvAndExpnssAmt), // rent - travel: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TrvCnfMtngRvAndExpnssAmt), - printing: formatBigInt(filing.IRS990PF.parts.pf_part_i?.PrntngAndPbNtInvstIncmAmt), - other: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthrExpnssRvAndExpnssAmt), - // ** - totalOperations: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TtOprExpnssRvAndExpnssAmt), - // ** - contributionsAndGrants: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CntrPdRvAndExpnssAmt), - total: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TtlExpnssRvAndExpnssAmt) - }), - + return { + ein, + year, + name: entityName, + city: filing.ReturnHeader.USAddrss_CtyNm, + state: filing.ReturnHeader.USAddrss_SttAbbrvtnCd, + website, + + revenue: _.pickBy({ + contributionsAndGrants: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CntrRcvdRvAndExpnssAmt), + interestOnSavings: formatBigInt(filing.IRS990PF.parts.pf_part_i?.IntrstOnSvRvAndExpnssAmt), + dividendsFromSecurities: formatBigInt(filing.IRS990PF.parts.pf_part_i?.DvdndsRvAndExpnssAmt), + netRental: formatBigInt(filing.IRS990PF.parts.pf_part_i?.NtRntlIncmOrLssAmt), + netAssetSales: formatBigInt(filing.IRS990PF.parts.pf_part_i?.NtGnSlAstRvAndExpnssAmt), + capitalGain: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CpGnNtIncmNtInvstIncmAmt), + capitalGainShortTerm: formatBigInt(filing.IRS990PF.parts.pf_part_i?.NtSTCptlGnAdjNtIncmAmt), + incomeModifications: formatBigInt(filing.IRS990PF.parts.pf_part_i?.IncmMdfctnsAdjNtIncmAmt), + grossSales: formatBigInt(filing.IRS990PF.parts.pf_part_i?.GrssPrftAdjNtIncmAmt), + other: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthrIncmRvAndExpnssAmt), + // ** + total: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TtlRvAndExpnssAmt) + }), + + expenses: _.pickBy({ + officerSalaries: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CmpOfcrDrTrstRvAndExpnssAmt), + nonOfficerSalaries: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthEmplSlrsWgsRvAndExpnssAmt), + employeeBenefits: formatBigInt(filing.IRS990PF.parts.pf_part_i?.PnsnEmplBnftRvAndExpnssAmt), + legalFees: formatBigInt(filing.IRS990PF.parts.pf_part_i?.LglFsRvAndExpnssAmt), + accountingFees: formatBigInt(filing.IRS990PF.parts.pf_part_i?.AccntngFsRvAndExpnssAmt), + otherProfessionalFees: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthrPrfFsRvAndExpnssAmt), + interest: formatBigInt(filing.IRS990PF.parts.pf_part_i?.IntrstRvAndExpnssAmt), + taxes: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TxsRvAndExpnssAmt), + depreciation: formatBigInt(filing.IRS990PF.parts.pf_part_i?.DprcAndDpltnRvAndExpnssAmt), + occupancy: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OccpncyRvAndExpnssAmt), // rent + travel: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TrvCnfMtngRvAndExpnssAmt), + printing: formatBigInt(filing.IRS990PF.parts.pf_part_i?.PrntngAndPbNtInvstIncmAmt), + other: formatBigInt(filing.IRS990PF.parts.pf_part_i?.OthrExpnssRvAndExpnssAmt), // ** - netIncome: formatBigInt(filing.IRS990PF.parts.pf_part_i?.ExcssRvnOvrExpnssAmt), - - assets: _.pickBy({ - cashBoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.CshBOYAmt), - cashEoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.CshEOYAmt), - boy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlAsstsBOYAmt), - eoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlAsstsEOYAmt) - }), - - liabilities: _.pickBy({ - boy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlLbltsBOYAmt), - eoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlLbltsEOYAmt) - }), - - netAssets: _.pickBy({ - boy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtNtAstOrFndBlncsBOYAmt), - eoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtNtAstOrFndBlncsEOYAmt) - }), - - applicantInfo: _.pickBy({ - acceptsUnsolicitedRequests: !filing.IRS990PF.parts.pf_part_xv?.OnlyCntrTPrslctdInd, - address: applicantSubmissionAddress, - recipientName: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_RcpntPrsnNm, - requirements: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_FrmAndInfAndMtrlsTxt, - deadlines: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_SbmssnDdlnsTxt, - restrictions: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_RstrctnsOnAwrdsTxt - }), - - directCharitableActivities: { - lineItems: _.filter(_.map(_.range(4), function (i) { - if (filing.IRS990PF.parts.pf_part_ixa?.[`Dscrptn${i}Txt`]) { - return { - description: filing.IRS990PF.parts.pf_part_ixa[`Dscrptn${i}Txt`], - expenses: formatBigInt(filing.IRS990PF.parts.pf_part_ixa[`Expnss${i}Amt`]) || 0 - } + totalOperations: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TtOprExpnssRvAndExpnssAmt), + // ** + contributionsAndGrants: formatBigInt(filing.IRS990PF.parts.pf_part_i?.CntrPdRvAndExpnssAmt), + total: formatBigInt(filing.IRS990PF.parts.pf_part_i?.TtlExpnssRvAndExpnssAmt) + }), + + // ** + netIncome: formatBigInt(filing.IRS990PF.parts.pf_part_i?.ExcssRvnOvrExpnssAmt), + + assets: _.pickBy({ + cashBoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.CshBOYAmt), + cashEoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.CshEOYAmt), + boy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlAsstsBOYAmt), + eoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlAsstsEOYAmt) + }), + + liabilities: _.pickBy({ + boy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlLbltsBOYAmt), + eoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtlLbltsEOYAmt) + }), + + netAssets: _.pickBy({ + boy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtNtAstOrFndBlncsBOYAmt), + eoy: formatBigInt(filing.IRS990PF.parts.pf_part_ii?.TtNtAstOrFndBlncsEOYAmt) + }), + + applicantInfo: _.pickBy({ + acceptsUnsolicitedRequests: !filing.IRS990PF.parts.pf_part_xv?.OnlyCntrTPrslctdInd, + address: applicantSubmissionAddress, + recipientName: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_RcpntPrsnNm, + requirements: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_FrmAndInfAndMtrlsTxt, + deadlines: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_SbmssnDdlnsTxt, + restrictions: filing.IRS990PF.groups.PFApplctnSbmssnInf?.[0]?.ApplctnSbmssnInf_RstrctnsOnAwrdsTxt + }), + + directCharitableActivities: { + lineItems: _.filter(_.map(_.range(4), function (i) { + if (filing.IRS990PF.parts.pf_part_ixa?.[`Dscrptn${i}Txt`]) { + return { + description: filing.IRS990PF.parts.pf_part_ixa[`Dscrptn${i}Txt`], + expenses: formatBigInt(filing.IRS990PF.parts.pf_part_ixa[`Expnss${i}Amt`]) || 0 } - })) - }, - - programRelatedInvestments: _.pickBy({ - lineItems: _.filter(_.map(_.range(4), function (i) { - if (filing.IRS990PF.parts.pf_part_ixb?.[`Dscrptn${i}Txt`]) { - return { - description: filing.IRS990PF.parts.pf_part_ixb[`Dscrptn${i}Txt`], - expenses: formatBigInt(filing.IRS990PF.parts.pf_part_ixb[`Expnss${i}Amt`]) || 0 - } + } + })) + }, + + programRelatedInvestments: _.pickBy({ + lineItems: _.filter(_.map(_.range(4), function (i) { + if (filing.IRS990PF.parts.pf_part_ixb?.[`Dscrptn${i}Txt`]) { + return { + description: filing.IRS990PF.parts.pf_part_ixb[`Dscrptn${i}Txt`], + expenses: formatBigInt(filing.IRS990PF.parts.pf_part_ixb[`Expnss${i}Amt`]) || 0 } - })), - otherTotal: formatBigInt(filing.IRS990PF.parts.pf_part_ixb?.AllOthrPrgrmRltdInvstTtAmt), - total: formatBigInt(filing.IRS990PF.parts.pf_part_ixb?.TtlAmt) - }) + } + })), + otherTotal: formatBigInt(filing.IRS990PF.parts.pf_part_ixb?.AllOthrPrgrmRltdInvstTtAmt), + total: formatBigInt(filing.IRS990PF.parts.pf_part_ixb?.TtlAmt) + }) - // TODO: could do some activities on whether or not they do political stuff (viia) - } - }, - - // 990pf - getFundJson (fund990, fundPersons, contributions, existing990s) { - const fund = { - ein: fund990.ein, - name: fund990.name, - city: fund990.city, - state: fund990.state, - website: fund990.website - } + // TODO: could do some activities on whether or not they do political stuff (viia) + } +} + +// 990pf +export function getFundJson (fund990, fundPersons, contributions, existing990s) { + const fund = { + ein: fund990.ein, + name: fund990.name, + city: fund990.city, + state: fund990.state, + website: fund990.website + } - const maxExistingYear = _.maxBy(existing990s, 'year')?.year - if ((fund990.year >= maxExistingYear) || !maxExistingYear) { - fund.maxYear = fund990.year - fund.assets = fund990.assets.eoy - fund.netAssetSales = fund990.netAssets.eoy - fund.liabilities = fund990.liabilities.eoy - - const grantAmounts = _.map(contributions, 'amount') - const hasGrants = grantAmounts.length > 0 - fund.lastYearStats = { - year: fund990.year, - revenue: fund990.revenue.total, - expenses: fund990.expenses.total, - grants: contributions.length, - grantSum: fund990.expenses.contributionsAndGrants, - grantMin: hasGrants ? _.min(grantAmounts) : 0, - grantMedian: hasGrants ? stats.median(grantAmounts) : 0, - grantMax: hasGrants ? _.max(grantAmounts) : 0 - } - - const contributionsWithNteeMajor = _.filter(contributions, ({ nteeMajor }) => nteeMajor && (nteeMajor !== '?')) - const nteeMajorGroups = _.groupBy(contributionsWithNteeMajor, 'nteeMajor') - fund.fundedNteeMajors = getStatsForContributionGroups(nteeMajorGroups, { - allContributions: contributionsWithNteeMajor - }) - - const nteeGroups = _.groupBy(contributionsWithNteeMajor, contribution => `${contribution.nteeMajor}${contribution.nteeMinor}`) - fund.fundedNtees = getStatsForContributionGroups(nteeGroups, { - allContributions: contributionsWithNteeMajor - }) - - const contributionsWithState = _.filter(contributions, 'toState') - const stateGroups = _.groupBy(contributionsWithState, 'toState') - fund.fundedStates = getStatsForContributionGroups(stateGroups, { - allContributions: contributionsWithState - }) - - fund.applicantInfo = fund990.applicantInfo - fund.directCharitableActivities = fund990.directCharitableActivities - fund.programRelatedInvestments = fund990.programRelatedInvestments + const maxExistingYear = _.maxBy(existing990s, 'year')?.year + if ((fund990.year >= maxExistingYear) || !maxExistingYear) { + fund.maxYear = fund990.year + fund.assets = fund990.assets.eoy + fund.netAssetSales = fund990.netAssets.eoy + fund.liabilities = fund990.liabilities.eoy + + const grantAmounts = _.map(contributions, 'amount') + const hasGrants = grantAmounts.length > 0 + fund.lastYearStats = { + year: fund990.year, + revenue: fund990.revenue.total, + expenses: fund990.expenses.total, + grants: contributions.length, + grantSum: fund990.expenses.contributionsAndGrants, + grantMin: hasGrants ? _.min(grantAmounts) : 0, + grantMedian: hasGrants ? stats.median(grantAmounts) : 0, + grantMax: hasGrants ? _.max(grantAmounts) : 0 } - return fund - }, - - getFundPersonsJson (filing) { - const entityName = getOrgNameByFiling(filing) - - return _.map(filing.IRS990PF.groups.PFOffcrDrTrstKyEmpl, function (person) { - let businessName = person.OffcrDrTrstKyEmpl_BsnssNmLn1 - if (person.OffcrDrTrstKyEmpl_BsnssNmLn2) { - businessName += ` ${person.OffcrDrTrstKyEmpl_BsnssNmLn2}` - } - return { - name: person.OffcrDrTrstKyEmpl_PrsnNm || businessName, - entityName, - entityType: 'fund', - year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), - isBusiness: Boolean(businessName), - title: person.OffcrDrTrstKyEmpl_TtlTxt, - weeklyHours: formatFloat(person.OffcrDrTrstKyEmpl_AvrgHrsPrWkDvtdTPsRt), - compensation: formatInt(person.OffcrDrTrstKyEmpl_CmpnstnAmt), - benefits: formatInt(person.OffcrDrTrstKyEmpl_EmplyBnftPrgrmAmt), - expenseAccount: formatInt(person.OffcrDrTrstKyEmpl_ExpnsAccntOthrAllwncAmt) - } + const contributionsWithNteeMajor = _.filter(contributions, ({ nteeMajor }) => nteeMajor && (nteeMajor !== '?')) + const nteeMajorGroups = _.groupBy(contributionsWithNteeMajor, 'nteeMajor') + fund.fundedNteeMajors = getStatsForContributionGroups(nteeMajorGroups, { + allContributions: contributionsWithNteeMajor }) - }, - - async getContributionsJson (filing) { - const contributions = _.map(filing.IRS990PF.groups.PFGrntOrCntrbtnPdDrYr, function (contribution) { - const city = contribution.RcpntUSAddrss_CtyNm - const state = contribution.RcpntUSAddrss_SttAbbrvtnCd - let businessName = contribution.RcpntBsnssNm_BsnssNmLn1Txt - if (contribution.RcpntBsnssNm_BsnssNmLn2Txt) { - businessName += ` ${contribution.RcpntBsnssNm_BsnssNmLn2Txt}` - } - - const type = businessName - ? 'org' - : contribution.GrntOrCntrbtnPdDrYr_RcpntPrsnNm - ? 'person' - : 'unknown' - - return { - year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), - toName: businessName || contribution.GrntOrCntrbtnPdDrYr_RcpntPrsnNm, - toCity: city, - toState: state, - type, - toExemptStatus: contribution.GrntOrCntrbtnPdDrYr_RcpntFndtnSttsTxt, - amount: formatBigInt(contribution.GrntOrCntrbtnPdDrYr_Amt), - relationship: contribution.GrntOrCntrbtnPdDrYr_RcpntRltnshpTxt, - purpose: contribution.GrntOrCntrbtnPdDrYr_GrntOrCntrbtnPrpsTxt - } + + const nteeGroups = _.groupBy(contributionsWithNteeMajor, contribution => `${contribution.nteeMajor}${contribution.nteeMinor}`) + fund.fundedNtees = getStatsForContributionGroups(nteeGroups, { + allContributions: contributionsWithNteeMajor }) - return await Promise.map(contributions, async function (contribution) { - const { year, toName, toCity, toState, purpose, amount } = contribution - const einNtee = await getEinNteeFromNameCityState(toName, toCity, toState) - const { ein, nteecc } = einNtee || {} - contribution = _.defaults({ - toId: ein || toName - }, contribution) - if (!contribution.toId) { - console.log('contribution missing toId', contribution) - } - if (nteecc) { - contribution.nteeMajor = nteecc.substr(0, 1) - contribution.nteeMinor = nteecc.substr(1) - } - - contribution.hash = md5([ - year, toName, toCity, toState, purpose, amount - ].join(':') - ) - - return contribution + const contributionsWithState = _.filter(contributions, 'toState') + const stateGroups = _.groupBy(contributionsWithState, 'toState') + fund.fundedStates = getStatsForContributionGroups(stateGroups, { + allContributions: contributionsWithState + }) + + fund.applicantInfo = fund990.applicantInfo + fund.directCharitableActivities = fund990.directCharitableActivities + fund.programRelatedInvestments = fund990.programRelatedInvestments + } + + return fund +} + +export function getFundPersonsJson (filing) { + const entityName = getOrgNameByFiling(filing) + + return _.map(filing.IRS990PF.groups.PFOffcrDrTrstKyEmpl, function (person) { + let businessName = person.OffcrDrTrstKyEmpl_BsnssNmLn1 + if (person.OffcrDrTrstKyEmpl_BsnssNmLn2) { + businessName += ` ${person.OffcrDrTrstKyEmpl_BsnssNmLn2}` + } + return { + name: person.OffcrDrTrstKyEmpl_PrsnNm || businessName, + entityName, + entityType: 'fund', + year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), + isBusiness: Boolean(businessName), + title: person.OffcrDrTrstKyEmpl_TtlTxt, + weeklyHours: formatFloat(person.OffcrDrTrstKyEmpl_AvrgHrsPrWkDvtdTPsRt), + compensation: formatInt(person.OffcrDrTrstKyEmpl_CmpnstnAmt), + benefits: formatInt(person.OffcrDrTrstKyEmpl_EmplyBnftPrgrmAmt), + expenseAccount: formatInt(person.OffcrDrTrstKyEmpl_ExpnsAccntOthrAllwncAmt) + } + }) +} + +export async function getContributionsJson (filing) { + const contributions = _.map(filing.IRS990PF.groups.PFGrntOrCntrbtnPdDrYr, function (contribution) { + const city = contribution.RcpntUSAddrss_CtyNm + const state = contribution.RcpntUSAddrss_SttAbbrvtnCd + let businessName = contribution.RcpntBsnssNm_BsnssNmLn1Txt + if (contribution.RcpntBsnssNm_BsnssNmLn2Txt) { + businessName += ` ${contribution.RcpntBsnssNm_BsnssNmLn2Txt}` + } + + const type = businessName + ? 'org' + : contribution.GrntOrCntrbtnPdDrYr_RcpntPrsnNm + ? 'person' + : 'unknown' + + return { + year: filing.ReturnHeader.RtrnHdr_TxPrdEndDt.substr(0, 4), + toName: businessName || contribution.GrntOrCntrbtnPdDrYr_RcpntPrsnNm, + toCity: city, + toState: state, + type, + toExemptStatus: contribution.GrntOrCntrbtnPdDrYr_RcpntFndtnSttsTxt, + amount: formatBigInt(contribution.GrntOrCntrbtnPdDrYr_Amt), + relationship: contribution.GrntOrCntrbtnPdDrYr_RcpntRltnshpTxt, + purpose: contribution.GrntOrCntrbtnPdDrYr_GrntOrCntrbtnPrpsTxt } - , { concurrency: 5 }) + }) + + return await Promise.map(contributions, async function (contribution) { + const { year, toName, toCity, toState, purpose, amount } = contribution + const einNtee = await getEinNteeFromNameCityState(toName, toCity, toState) + const { ein, nteecc } = einNtee || {} + contribution = _.defaults({ + toId: ein || toName + }, contribution) + if (!contribution.toId) { + console.log('contribution missing toId', contribution) + } + if (nteecc) { + contribution.nteeMajor = nteecc.substr(0, 1) + contribution.nteeMinor = nteecc.substr(1) + } + + contribution.hash = md5([ + year, toName, toCity, toState, purpose, amount + ].join(':') + ) + + return contribution } + , { concurrency: 5 }) } function getStatsForContributionGroups (contributionGroups, { allContributions }) { diff --git a/services/irs_990_importer/helpers.js b/services/irs_990_importer/helpers.js index 95bb870..d3f20a8 100644 --- a/services/irs_990_importer/helpers.js +++ b/services/irs_990_importer/helpers.js @@ -2,56 +2,54 @@ import normalizeUrl from 'normalize-url' import _ from 'lodash' import { cknex } from 'backend-shared' -export default { - formatInt (int) { - if (int != null) { - return parseInt(int) - } else { - return null - } - }, - // cassanknex doesn't use v4 of cassandra-driver which supports `BigInt`, so have to use Long - formatBigInt (bigint) { - if (bigint != null) { - return cknex.Long.fromValue(bigint) - } else { - return null - } - }, - formatFloat (float) { - if (float != null) { - return parseFloat(float) - } else { - return null - } - }, - formatWebsite (website) { - if (website && (website !== 'N/A')) { - try { - website = normalizeUrl(website) - } catch (err) {} - } - return website - }, - getOrgNameByFiling (filing) { - let entityName = filing.ReturnHeader.BsnssNm_BsnssNmLn1Txt - if (filing.ReturnHeader.BsnssNm_BsnssNmLn2Txt) { - entityName += ` ${filing.ReturnHeader.BsnssNm_BsnssNmLn2Txt}` - } - return entityName - }, +export function formatInt (int) { + if (int != null) { + return parseInt(int) + } else { + return null + } +} +// cassanknex doesn't use v4 of cassandra-driver which supports `BigInt`, so have to use Long +export function formatBigInt (bigint) { + if (bigint != null) { + return cknex.Long.fromValue(bigint) + } else { + return null + } +} +export function formatFloat (float) { + if (float != null) { + return parseFloat(float) + } else { + return null + } +} +export function formatWebsite (website) { + if (website && (website !== 'N/A')) { + try { + website = normalizeUrl(website) + } catch (err) {} + } + return website +} +export function getOrgNameByFiling (filing) { + let entityName = filing.ReturnHeader.BsnssNm_BsnssNmLn1Txt + if (filing.ReturnHeader.BsnssNm_BsnssNmLn2Txt) { + entityName += ` ${filing.ReturnHeader.BsnssNm_BsnssNmLn2Txt}` + } + return entityName +} - roundTwoDigits (num) { - return Math.round(num * 100) / 100 - }, +export function roundTwoDigits (num) { + return Math.round(num * 100) / 100 +} - sumByLong (arr, key) { - return _.reduce(arr, function (long, row) { - if (row[key]) { - long = long.add(row[key]) - } - return long +export function sumByLong (arr, key) { + return _.reduce(arr, function (long, row) { + if (row[key]) { + long = long.add(row[key]) } - , cknex.Long.fromValue(0)) + return long } + , cknex.Long.fromValue(0)) } diff --git a/services/irs_990_importer/index.js b/services/irs_990_importer/index.js index 3f28dec..f690bba 100644 --- a/services/irs_990_importer/index.js +++ b/services/irs_990_importer/index.js @@ -5,135 +5,124 @@ import { JobCreate } from 'backend-shared' import IrsFund from '../../graphql/irs_fund/model.js' import IrsFund990 from '../../graphql/irs_fund_990/model.js' import IrsOrg990 from '../../graphql/irs_org_990/model.js' -import JobService from '../../services/job.js' +import * as JobService from '../../services/job.js' import config from '../../config.js' // FIXME: classify community foundatoins (990 instead of 990pf) as fund and org? -class Irs990Service { - constructor () { - this.processUnprocessed = this.processUnprocessed.bind(this) - this.processUnprocessedOrgs = this.processUnprocessedOrgs.bind(this) - this.processUnprocessedFunds = this.processUnprocessedFunds.bind(this) - } - - processEin = async (ein, { type }) => { - const Model990 = type === 'fund' ? IrsFund990 : IrsOrg990 - const chunk = await Model990.getAllByEin(ein) - return JobCreate.createJob({ - queue: JobService.QUEUES.DEFAULT, - waitForCompletion: true, - job: { chunk }, - type: type === 'fund' - ? JobService.TYPES.DEFAULT.IRS_990_PROCESS_FUND_CHUNK - : JobService.TYPES.DEFAULT.IRS_990_PROCESS_ORG_CHUNK, - ttlMs: 120000, - priority: JobService.PRIORITIES.NORMAL - }) - }; +export async function processEin (ein, { type }) { + const Model990 = type === 'fund' ? IrsFund990 : IrsOrg990 + const chunk = await Model990.getAllByEin(ein) + return JobCreate.createJob({ + queue: JobService.QUEUES.DEFAULT, + waitForCompletion: true, + job: { chunk }, + type: type === 'fund' + ? JobService.TYPES.DEFAULT.IRS_990_PROCESS_FUND_CHUNK + : JobService.TYPES.DEFAULT.IRS_990_PROCESS_ORG_CHUNK, + ttlMs: 120000, + priority: JobService.PRIORITIES.NORMAL + }) +} - // some large funds need to be processed 1 by 1 to not overload scylla - fixBadFundImports = async ({ limit = 10000 }) => { - const irsFunds = await IrsFund.search({ - trackTotalHits: true, - limit, - query: { - bool: { - must: [ - { - range: { - 'lastYearStats.grants': { gt: 1 } - } - }, - { - range: { - assets: { gt: 100000000 } - } +// some large funds need to be processed 1 by 1 to not overload scylla +export async function fixBadFundImports ({ limit = 10000 }) { + const irsFunds = await IrsFund.search({ + trackTotalHits: true, + limit, + query: { + bool: { + must: [ + { + range: { + 'lastYearStats.grants': { gt: 1 } } - ] - } - } - }) - console.log(`Fetched ${irsFunds.rows.length} / ${irsFunds.total}`) - return Promise.each(irsFunds.rows, irsFund => { - return this.processEin(irsFund.ein, { type: 'fund' }) - }) - .then(() => console.log('done all')) - }; - - async processUnprocessed (options) { - const { - limit = 6000, chunkSize = 300, chunkConcurrency, recursive, - Model990, jobType - } = options - let start = Date.now() - const model990s = await Model990.search({ - trackTotalHits: true, - limit, - query: { - bool: { - // should: _.map config.VALID_RETURN_VERSIONS, (version) -> - // match: returnVersion: version - must: { + }, + { range: { - // 'assets.eoy': gt: 100000000 - importVersion: { lt: config.CURRENT_IMPORT_VERSION } + assets: { gt: 100000000 } } } - } + ] } - }) - console.log(`Fetched ${model990s.rows.length} / ${model990s.total} in ${Date.now() - start} ms`) - start = Date.now() - const chunks = _.chunk(model990s.rows, chunkSize) - await Promise.map(chunks, chunk => { - return JobCreate.createJob({ - queue: JobService.QUEUES.DEFAULT, - waitForCompletion: true, - job: { chunk, chunkConcurrency }, - type: jobType, - ttlMs: 120000, - priority: JobService.PRIORITIES.NORMAL - }) - .catch(err => console.log('err', err)) - }) + } + }) + console.log(`Fetched ${irsFunds.rows.length} / ${irsFunds.total}`) + return Promise.each(irsFunds.rows, irsFund => { + return processEin(irsFund.ein, { type: 'fund' }) + }) + .then(() => console.log('done all')) +} - if (model990s.total) { - console.log(`Finished step (${limit}) in ${Date.now() - start} ms`) - await Model990.refreshESIndex() - if (recursive) { - return this.processUnprocessed(options) +export async function processUnprocessed (options) { + const { + limit = 6000, chunkSize = 300, chunkConcurrency, recursive, + Model990, jobType + } = options + let start = Date.now() + const model990s = await Model990.search({ + trackTotalHits: true, + limit, + query: { + bool: { + // should: _.map config.VALID_RETURN_VERSIONS, (version) -> + // match: returnVersion: version + must: { + range: { + // 'assets.eoy': gt: 100000000 + importVersion: { lt: config.CURRENT_IMPORT_VERSION } + } + } } - } else { - return console.log('done') } - } - - processUnprocessedOrgs ({ limit, chunkSize, chunkConcurrency, recursive }) { - return this.processUnprocessed({ - limit, - chunkSize, - chunkConcurrency, - recursive, - Model990: IrsOrg990, - jobType: JobService.TYPES.DEFAULT.IRS_990_PROCESS_ORG_CHUNK + }) + console.log(`Fetched ${model990s.rows.length} / ${model990s.total} in ${Date.now() - start} ms`) + start = Date.now() + const chunks = _.chunk(model990s.rows, chunkSize) + await Promise.map(chunks, chunk => { + return JobCreate.createJob({ + queue: JobService.QUEUES.DEFAULT, + waitForCompletion: true, + job: { chunk, chunkConcurrency }, + type: jobType, + ttlMs: 120000, + priority: JobService.PRIORITIES.NORMAL }) - } + .catch(err => console.log('err', err)) + }) - processUnprocessedFunds ({ limit, chunkSize, chunkConcurrency, recursive }) { - return this.processUnprocessed({ - limit, - chunkSize, - chunkConcurrency, - recursive, - Model990: IrsFund990, - jobType: JobService.TYPES.DEFAULT.IRS_990_PROCESS_FUND_CHUNK - }) + if (model990s.total) { + console.log(`Finished step (${limit}) in ${Date.now() - start} ms`) + await Model990.refreshESIndex() + if (recursive) { + return processUnprocessed(options) + } + } else { + return console.log('done') } } -export default new Irs990Service() +export function processUnprocessedOrgs ({ limit, chunkSize, chunkConcurrency, recursive }) { + return processUnprocessed({ + limit, + chunkSize, + chunkConcurrency, + recursive, + Model990: IrsOrg990, + jobType: JobService.TYPES.DEFAULT.IRS_990_PROCESS_ORG_CHUNK + }) +} +export function processUnprocessedFunds ({ limit, chunkSize, chunkConcurrency, recursive }) { + return processUnprocessed({ + limit, + chunkSize, + chunkConcurrency, + recursive, + Model990: IrsFund990, + jobType: JobService.TYPES.DEFAULT.IRS_990_PROCESS_FUND_CHUNK + }) +} /* truncate irs_990_api.irs_orgs_by_ein truncate irs_990_api.irs_orgs_990_by_ein_and_year diff --git a/services/irs_990_importer/load_all_for_year.js b/services/irs_990_importer/load_all_for_year.js index d0d24be..b4b387b 100644 --- a/services/irs_990_importer/load_all_for_year.js +++ b/services/irs_990_importer/load_all_for_year.js @@ -14,63 +14,61 @@ function getIndexJson (year) { return request(indexUrl) } -export default { - async loadAllForYear (year) { - let index - if (year) { - index = JSON.parse(await getIndexJson(year)) - } else { - index = await import('../../data/sample_index.json') - year = 2016 - } +export async function loadAllForYear (year) { + let index + if (year) { + index = JSON.parse(await getIndexJson(year)) + } else { + index = await import('../../data/sample_index.json') + year = 2016 + } - console.log('got index') - console.log('keys', _.keys(index)) - const filings = index[`Filings${year}`] - console.log(filings.length) - const chunks = _.chunk(filings, 500) - return Promise.map(chunks, function (chunk, i) { - const funds = _.filter(chunk, { FormType: '990PF' }) - const orgs = _.filter(chunk, ({ FormType }) => FormType !== '990PF') - console.log(i * 100) - console.log('funds', funds.length, 'orgs', orgs.length) - return Promise.all(_.filter([ - funds.length && - IrsFund.batchUpsert(_.map(funds, filing => ({ - ein: filing.EIN, - name: filing.OrganizationName - }))), - funds.length && - IrsFund990.batchUpsert(_.map(funds, filing => ({ - ein: filing.EIN, - year: filing.TaxPeriod.substr(0, 4), - taxPeriod: filing.TaxPeriod, - objectId: filing.ObjectId, - submitDate: new Date(filing.SubmittedOn), - lastIrsUpdate: new Date(filing.LastUpdated), - type: filing.FormType, - xmlUrl: filing.URL - }))), + console.log('got index') + console.log('keys', _.keys(index)) + const filings = index[`Filings${year}`] + console.log(filings.length) + const chunks = _.chunk(filings, 500) + return Promise.map(chunks, function (chunk, i) { + const funds = _.filter(chunk, { FormType: '990PF' }) + const orgs = _.filter(chunk, ({ FormType }) => FormType !== '990PF') + console.log(i * 100) + console.log('funds', funds.length, 'orgs', orgs.length) + return Promise.all(_.filter([ + funds.length && + IrsFund.batchUpsert(_.map(funds, filing => ({ + ein: filing.EIN, + name: filing.OrganizationName + }))), + funds.length && + IrsFund990.batchUpsert(_.map(funds, filing => ({ + ein: filing.EIN, + year: filing.TaxPeriod.substr(0, 4), + taxPeriod: filing.TaxPeriod, + objectId: filing.ObjectId, + submitDate: new Date(filing.SubmittedOn), + lastIrsUpdate: new Date(filing.LastUpdated), + type: filing.FormType, + xmlUrl: filing.URL + }))), - orgs.length && - IrsOrg.batchUpsert(_.map(orgs, filing => ({ - ein: filing.EIN, - name: filing.OrganizationName - }))), - orgs.length && - IrsOrg990.batchUpsert(_.map(orgs, filing => ({ - ein: filing.EIN, - year: filing.TaxPeriod.substr(0, 4), - taxPeriod: filing.TaxPeriod, - objectId: filing.ObjectId, - submitDate: new Date(filing.SubmittedOn), - lastIrsUpdate: new Date(filing.LastUpdated), - type: filing.FormType, - xmlUrl: filing.URL - }))) - ])) - } - , { concurrency: 10 }) - .then(() => console.log('done')) + orgs.length && + IrsOrg.batchUpsert(_.map(orgs, filing => ({ + ein: filing.EIN, + name: filing.OrganizationName + }))), + orgs.length && + IrsOrg990.batchUpsert(_.map(orgs, filing => ({ + ein: filing.EIN, + year: filing.TaxPeriod.substr(0, 4), + taxPeriod: filing.TaxPeriod, + objectId: filing.ObjectId, + submitDate: new Date(filing.SubmittedOn), + lastIrsUpdate: new Date(filing.LastUpdated), + type: filing.FormType, + xmlUrl: filing.URL + }))) + ])) } + , { concurrency: 10 }) + .then(() => console.log('done')) } diff --git a/services/irs_990_importer/ntee.js b/services/irs_990_importer/ntee.js index 2118a61..5c038a5 100644 --- a/services/irs_990_importer/ntee.js +++ b/services/irs_990_importer/ntee.js @@ -5,61 +5,59 @@ import { Cache } from 'backend-shared' import IrsOrg from '../../graphql/irs_org/model.js' import CacheService from '../../services/cache.js' -export default { - getEinNteeFromNameCityState (name, city, state) { - name = name?.toLowerCase() || '' - city = city?.toLowerCase() || '' - state = state?.toLowerCase() || '' - const key = `${CacheService.PREFIXES.EIN_FROM_NAME}:${name}:${city}:${state}` - return Cache.preferCache(key, async function () { - const orgs = await IrsOrg.search({ - limit: 10, - query: { - multi_match: { - query: name, - type: 'bool_prefix', - fields: ['name', 'name._2gram'] - } +export function getEinNteeFromNameCityState (name, city, state) { + name = name?.toLowerCase() || '' + city = city?.toLowerCase() || '' + state = state?.toLowerCase() || '' + const key = `${CacheService.PREFIXES.EIN_FROM_NAME}:${name}:${city}:${state}` + return Cache.preferCache(key, async function () { + const orgs = await IrsOrg.search({ + limit: 10, + query: { + multi_match: { + query: name, + type: 'bool_prefix', + fields: ['name', 'name._2gram'] } - }) - - const closeEnough = _.filter(_.map(orgs.rows, (org) => { - if (!org.name) { - return 0 - } - const score = stringSimilarity.compareTwoStrings(org.name.toLowerCase(), name) - // console.log score - if (score > 0.7) { - return _.defaults({ score }, org) - } - })) - const cityMatches = _.filter(_.map(closeEnough, (org) => { - let cityScore - if (!org.city) { - return 0 - } - if (city) { - cityScore = stringSimilarity.compareTwoStrings(org.city.toLowerCase(), city) - } else { - cityScore = 1 - } - if (cityScore > 0.8) { - return _.defaults({ cityScore: city }, org) - } - })) - - let match = _.maxBy(cityMatches, ({ score, cityScore }) => `${cityScore}|${score}`) - if (!match) { - match = _.maxBy(closeEnough, 'score') } + }) - if (match) { - return { ein: match?.ein, nteecc: match?.nteecc } + const closeEnough = _.filter(_.map(orgs.rows, (org) => { + if (!org.name) { + return 0 + } + const score = stringSimilarity.compareTwoStrings(org.name.toLowerCase(), name) + // console.log score + if (score > 0.7) { + return _.defaults({ score }, org) + } + })) + const cityMatches = _.filter(_.map(closeEnough, (org) => { + let cityScore + if (!org.city) { + return 0 + } + if (city) { + cityScore = stringSimilarity.compareTwoStrings(org.city.toLowerCase(), city) } else { - return null + cityScore = 1 + } + if (cityScore > 0.8) { + return _.defaults({ cityScore: city }, org) } + })) + + let match = _.maxBy(cityMatches, ({ score, cityScore }) => `${cityScore}|${score}`) + if (!match) { + match = _.maxBy(closeEnough, 'score') + } + + if (match) { + return { ein: match?.ein, nteecc: match?.nteecc } + } else { + return null } - // TODO: can also look at grant amount and income to help find best match - , { expireSeconds: 3600 * 24 }) } + // TODO: can also look at grant amount and income to help find best match + , { expireSeconds: 3600 * 24 }) } diff --git a/services/irs_990_importer/parse_websites.js b/services/irs_990_importer/parse_websites.js index bc49c46..df88f2f 100644 --- a/services/irs_990_importer/parse_websites.js +++ b/services/irs_990_importer/parse_websites.js @@ -2,7 +2,7 @@ import _ from 'lodash' import { JobCreate } from 'backend-shared' import IrsOrg from '../../graphql/irs_org/model.js' -import JobService from '../../services/job.js' +import * as JobService from '../../services/job.js' export const parseGrantMakingWebsites = async () => { const { rows } = await IrsOrg.search({ diff --git a/services/irs_990_importer/set_ntee.js b/services/irs_990_importer/set_ntee.js index 82784a0..15fbb6f 100644 --- a/services/irs_990_importer/set_ntee.js +++ b/services/irs_990_importer/set_ntee.js @@ -2,50 +2,48 @@ import requestNonPromise from 'request' import csv from 'csvtojson' import fs from 'fs' import { JobCreate } from 'backend-shared' -import IrsOrg from '../../graphql/irs_org/model' -import JobService from '../../services/job.js' +import IrsOrg from '../../graphql/irs_org/model.js' +import * as JobService from '../../services/job.js' import config from '../../config.js' -export default { - setNtee () { - console.log('sync') - let cache = null - return requestNonPromise(config.NTEE_CSV) - .pipe(fs.createWriteStream('data.csv')) - .on('finish', function () { - console.log('file downloaded') - let chunk = [] - let i = 0 - return csv().fromFile('data.csv') - .subscribe(function (json) { - i += 1 - // batch every 100 for upsert - if (i && !(i % 100)) { - console.log(i) - cache = chunk - chunk = [] - JobCreate.createJob({ - queue: JobService.QUEUES.DEFAULT, - waitForCompletion: true, - job: { orgs: cache, i }, - type: JobService.TYPES.DEFAULT.IRS_990_UPSERT_ORGS, - ttlMs: 60000, - priority: JobService.PRIORITIES.NORMAL - }).catch(err => console.log('err', err)) - } +export function setNtee () { + console.log('sync') + let cache = null + return requestNonPromise(config.NTEE_CSV) + .pipe(fs.createWriteStream('data.csv')) + .on('finish', function () { + console.log('file downloaded') + let chunk = [] + let i = 0 + return csv().fromFile('data.csv') + .subscribe(function (json) { + i += 1 + // batch every 100 for upsert + if (i && !(i % 100)) { + console.log(i) + cache = chunk + chunk = [] + JobCreate.createJob({ + queue: JobService.QUEUES.DEFAULT, + waitForCompletion: true, + job: { orgs: cache, i }, + type: JobService.TYPES.DEFAULT.IRS_990_UPSERT_ORGS, + ttlMs: 60000, + priority: JobService.PRIORITIES.NORMAL + }).catch(err => console.log('err', err)) + } - return chunk.push({ - ein: json.EIN, - name: json.NAME, - city: json.CITY, - state: json.STATE, - nteecc: json.NTEECC - }) - }, () => console.log('error'), function () { - console.log('done') - return IrsOrg.batchUpsert(cache) + return chunk.push({ + ein: json.EIN, + name: json.NAME, + city: json.CITY, + state: json.STATE, + nteecc: json.NTEECC }) - }) - } + }, () => console.log('error'), function () { + console.log('done') + return IrsOrg.batchUpsert(cache) + }) + }) } diff --git a/services/setup.js b/services/setup.js index bf98787..a1f1027 100644 --- a/services/setup.js +++ b/services/setup.js @@ -1,28 +1,44 @@ import fs from 'fs' import _ from 'lodash' import Promise from 'bluebird' -import { cknex, ElasticsearchSetup, JobRunner, ScyllaSetup } from 'backend-shared' +import { + cknex, elasticsearch, ElasticsearchSetup, JobRunner, ScyllaSetup, Cache, PubSub +} from 'backend-shared' import config from '../config.js' import { RUNNERS } from './job.js' +function sharedSetup () { + Cache.setup({ + prefix: config.REDIS.PREFIX, + cacheHost: config.REDIS.CACHE_HOST, + persistentHost: config.REDIS.PERSISTENT_HOST, + port: config.REDIS.port + }) + cknex.setup('irs_990_api', config.SCYLLA.CONTACT_POINTS) + elasticsearch.setup(`${config.ELASTICSEARCH.HOST}:9200`) + PubSub.setup(config.REDIS.PUB_SUB_HOST, config.REDIS.PORT, config.REDIS.PUB_SUB_PREFIX) +} + async function setup () { - cknex.setDefaultKeyspace('irs_990_api') + sharedSetup() const graphqlFolders = _.filter(fs.readdirSync('./graphql'), file => file.indexOf('.') === -1) - const scyllaTables = await Promise.map(graphqlFolders, async (folder) => { - const model = await import(`../graphql/${folder}/model`) - return model?.getScyllaTables?.() || [] - }) - const elasticSearchIndices = await Promise.map(graphqlFolders, async (folder) => { - const model = await import(`../graphql/${folder}/model`) - return model?.getElasticSearchIndices?.() || [] - }) + const scyllaTables = _.flatten(await Promise.map(graphqlFolders, async (folder) => { + const model = await import(`../graphql/${folder}/model.js`) + return model?.default?.getScyllaTables?.() || [] + })) + const elasticSearchIndices = _.flatten(await Promise.map(graphqlFolders, async (folder) => { + const model = await import(`../graphql/${folder}/model.js`) + + return model?.default?.getElasticSearchIndices?.() || [] + })) - const shouldRunSetup = true || (config.get().ENV === config.get().ENVS.PRODUCTION) || - (config.get().SCYLLA.CONTACT_POINTS[0] === 'localhost') + const isDev = config.ENV === config.ENVS.DEV + const shouldRunSetup = true || (config.ENV === config.ENVS.PRODUCTION) || + (config.SCYLLA.CONTACT_POINTS[0] === 'localhost') await Promise.all(_.filter([ - shouldRunSetup && ScyllaSetup.setup(scyllaTables) + shouldRunSetup && ScyllaSetup.setup(scyllaTables, {isDev}) .then(() => console.log('scylla setup')), shouldRunSetup && ElasticsearchSetup.setup(elasticSearchIndices) .then(() => console.log('elasticsearch setup')) @@ -35,7 +51,7 @@ async function setup () { } function childSetup () { - cknex.setDefaultKeyspace('irs_990_api') + sharedSetup() JobRunner.listen(RUNNERS) cknex.enableErrors() return Promise.resolve(null) // don't block