- limits

- better es schema
- /processEin for funds

config.coffee

@@ -4,7 +4,7 @@ assertNoneMissing = require 'assert-none-missing'
 env = process.env
 
 config =
-  CURRENT_IMPORT_VERSION: 15 # increment any time you want to repull all data
+  CURRENT_IMPORT_VERSION: 16 # increment any time you want to repull all data
   VALID_RETURN_VERSIONS: [
     # https://github.com/breezyhq/990-xml-reader/blob/master/irs_reader/settings.py#L36
     '2013v3.0', '2013v3.1', '2013v4.0', '2014v5.0', '2014v6.0',

graphql/irs_contribution/model.coffee

@@ -67,11 +67,26 @@ class IrsContributionModel extends Base
       }
     ]
 
-  getAllByFromEin: (fromEin) =>
-    cknex().select '*'
+  getAllByFromEin: (fromEin, {limit} = {}) =>
+    q = cknex().select '*'
     .from 'irs_contributions_by_fromEin_and_toId'
     .where 'fromEin', '=', fromEin
-    .run()
+
+    if limit
+      q.limit limit
+
+    q.run()
+    .map @defaultOutput
+
+  getAllByToId: (toId, {limit} = {}) =>
+    q = cknex().select '*'
+    .from 'irs_contributions_by_toId'
+    .where 'toId', '=', toId
+
+    if limit
+      q.limit limit
+
+    q.run()
     .map @defaultOutput
 
 module.exports = new IrsContributionModel()

graphql/irs_contribution/resolvers.coffee

@@ -11,16 +11,20 @@ irsContributionLoader = Loader.withContext (ids, context) ->
 
 module.exports = {
   Query:
-    irsContributions: (_, {fromEin, limit}) ->
-      IrsContribution.getAllByFromEin fromEin, {limit}
-      .then GraphqlFormatter.fromScylla
+    irsContributions: (_, {fromEin, toId, limit}) ->
+      if fromEin
+        IrsContribution.getAllByFromEin fromEin, {limit}
+        .then GraphqlFormatter.fromScylla
+      else if toId
+        IrsContribution.getAllByToId toId, {limit}
+        .then GraphqlFormatter.fromScylla
 
   IrsContribution:
     __resolveReference: (irsContribution) ->
       irsContributionLoader(context).load irsContribution.id
 
   IrsFund:
-    irsContributions: (irsFund) ->
-      IrsContribution.getAllByFromEin irsFund.ein
+    irsContributions: (irsFund, {limit}) ->
+      IrsContribution.getAllByFromEin irsFund.ein, {limit}
       .then GraphqlFormatter.fromScylla
 }

graphql/irs_contribution/type.graphql

@@ -24,5 +24,5 @@ extend type IrsFund {
 }
 
 extend type Query {
-  irsContributions(fromEin: String, limit: Int): IrsContributionConnection
+  irsContributions(fromEin: String, toId: String, limit: Int): IrsContributionConnection
 }

graphql/irs_fund/model.coffee

@@ -28,9 +28,9 @@ class IrsFundModel extends Base
           netAssets: 'bigint'
           liabilities: 'bigint'
 
-          lastRevenue: 'bigint'
-          lastExpenses: 'bigint'
-          lastContributionsAndGrants: 'bigint'
+          lastYearStats: 'json'
+          # {<nteeMajor>: {count, percent}}
+          fundedNteeMajors: 'json'
         primaryKey:
           partitionKey: ['ein']
       }
@@ -51,16 +51,50 @@ class IrsFundModel extends Base
           mission: {type: 'text'}
           exemptStatus: {type: 'text'}
 
+          applicantInfo:
+            properties:
+              acceptsUnsolicitedRequests: {type: 'boolean'}
+              address:
+                properties:
+                  street1: {type: 'keyword'}
+                  street2: {type: 'keyword'}
+                  postalCode: {type: 'keyword'}
+                  city: {type: 'keyword'}
+                  state: {type: 'keyword'}
+                  countryCode: {type: 'keyword'}
+              recipientName: {type: 'text'}
+              requirements: {type: 'text'}
+              deadlines: {type: 'text'}
+              restrictions: {type: 'text'}
+          directCharitableActivities:
+            properties:
+              lineItem:
+                properties:
+                  description: {type: 'text'}
+                  expenses: {type: 'long'}
+          programRelatedInvestments:
+            properties:
+              lineItem:
+                properties:
+                  description: {type: 'text'}
+                  expenses: {type: 'long'}
+
           assets: {type: 'long'}
           netAssets: {type: 'long'}
           liabilities: {type: 'long'}
-          lastRevenue: {type: 'long'}
-          lastExpenses: {type: 'long'}
-          lastContributionsAndGrants: {type: 'long'}
 
-          applicantInfo: {type: 'object'}
-          directCharitableActivities: {type: 'object'}
-          programRelatedInvestments: {type: 'object'}
+          lastYearStats:
+            properties:
+              year: {type: 'integer'}
+              revenue: {type: 'long'}
+              expenses: {type: 'long'}
+              grants: {type: 'integer'}
+              grantSum: {type: 'long'}
+              grantMin: {type: 'integer'}
+              grantMedian: {type: 'float'}
+              grantMax: {type: 'integer'}
+
+          fundedNteeMajors: {type: 'object'}
 
           websiteText: {type: 'text'} # TODO: move to diff table?
       }

graphql/irs_fund/type.graphql

@@ -1,7 +1,7 @@
 type IrsFund {
   ein: String
   name: String @nameCase
-  city: String
+  city: String @nameCase
   state: String # 2 letter code
   nteecc: String # https://nccs.urban.org/project/national-taxonomy-exempt-entities-ntee-codes
 
@@ -17,9 +17,8 @@ type IrsFund {
   netAssets: BigInt
   liabilities: BigInt
 
-  lastRevenue: BigInt
-  lastExpenses: BigInt
-  lastContributionsAndGrants: BigInt
+  lastYearStats: LastYearStats
+  fundedNteeMajors: JSONObject # {<nteeMajor>: {percent, count}}
 }
 
 type ApplicantInfo {
@@ -49,6 +48,17 @@ type ProgramRelatedInvestmentLineItem {
   expenses: BigInt
 }
 
+type LastYearStats {
+  year: Int
+  revenue: BigInt
+  expenses: BigInt
+  grants: Int
+  grantSum: BigInt
+  grantMin: Int
+  grantMedian: Float
+  grantMax: Int
+}
+
 type IrsFundConnection {
   nodes: [IrsFund!]
   totalCount: Int

graphql/irs_org/model.coffee

@@ -58,6 +58,7 @@ class IrsOrgModel extends Base
           lastRevenue: {type: 'long'}
           lastExpenses: {type: 'long'}
 
+          # TODO: specify properties & reindex
           topSalary: {type: 'object'}
 
           websiteText: {type: 'text'} # TODO: move to diff table?

graphql/irs_org/type.graphql

@@ -18,6 +18,8 @@ type IrsOrg {
   lastRevenue: BigInt
   lastExpenses: BigInt
 
+  # TODO: switch over to lastYearStats like funds
+
   topSalary: JSONObject
   websiteText: String
 }

graphql/irs_org_990/model.coffee

@@ -83,6 +83,7 @@ class IrsOrg990Model extends Base
           employeeCount: {type: 'integer'}
           volunteerCount: {type: 'integer'}
 
+          # TODO specify properties & reindex
           revenue: {type: 'object'}
           expenses: {type: 'object'}
           assets: {type: 'object'}

graphql/irs_org_990/type.graphql

@@ -22,10 +22,10 @@ type IrsOrg990 {
   volunteerCount: Int
 
   revenue: IrsOrg990Revenue
-  expenses: String
-  assets: String
-  liabilities: String
-  netAssets: String
+  expenses: String # TODO
+  assets: String # TODO
+  liabilities: String # TODO
+  netAssets: String # TODO
 }
 
 type IrsOrg990Revenue {

index.coffee

@@ -148,7 +148,7 @@ app.get '/processUnprocessedOrgs', (req, res) ->
 
 app.get '/processEin', (req, res) ->
   {processEin} = require './services/irs_990_importer'
-  processEin req.query.ein
+  processEin req.query.ein, {type: req.query.type}
   res.send 'processing org'
 
 # chunkConcurrency=10

package.json

@@ -55,6 +55,7 @@
     "redlock": "^2.0.0",
     "request": "^2.88.2",
     "request-promise": "^4.1.1",
+    "stats-lite": "^2.2.0",
     "string-similarity": "^3.0.0"
   }
 }

services/irs_990_importer/format_irs_990pf.coffee

@@ -1,8 +1,10 @@
 _ = require 'lodash'
 Promise = require 'bluebird'
 md5 = require 'md5'
+stats = require 'stats-lite'
 
-{formatInt, formatBigInt, formatWebsite, formatFloat, getOrgNameByFiling} = require './helpers'
+{formatInt, formatBigInt, formatWebsite, formatFloat,
+  roundTwoDigits, getOrgNameByFiling} = require './helpers'
 {getEinNteeFromNameCityState} = require './ntee'
 
 module.exports = {
@@ -130,16 +132,31 @@ module.exports = {
       website: fund990.website
     }
 
-    maxExistingYear = _.maxBy existing990s, 'year'
+    maxExistingYear = _.maxBy(existing990s, 'year')?.year
     if fund990.year >= maxExistingYear or not maxExistingYear
       fund.maxYear = fund990.year
       fund.assets = fund990.assets.eoy
       fund.netAssetSales = fund990.netAssets.eoy
       fund.liabilities = fund990.liabilities.eoy
 
-      fund.lastRevenue = fund990.revenue.total
-      fund.lastExpenses = fund990.expenses.total
-      fund.lastContributionsAndGrants = fund990.expenses.contributionsAndGrants
+      grantAmounts = _.map contributions, 'amount'
+      hasGrants = grantAmounts.length > 0
+      fund.lastYearStats =
+        year: fund990.year
+        revenue: fund990.revenue.total
+        expenses: fund990.expenses.total
+        grants: contributions.length
+        grantSum: fund990.expenses.contributionsAndGrants
+        grantMin: if hasGrants then _.min grantAmounts else 0
+        grantMedian: if hasGrants then stats.median grantAmounts else 0
+        grantMax: if hasGrants then _.max grantAmounts else 0
+
+      contributionsWithNteeMajor = _.filter contributions, ({nteeMajor}) ->
+        nteeMajor and nteeMajor isnt '?'
+      nteeCounts = _.countBy _.map contributionsWithNteeMajor, 'nteeMajor'
+      fund.fundedNteeMajors = _.mapValues nteeCounts, (count) ->
+        percent = roundTwoDigits 100 * count / contributionsWithNteeMajor.length
+        {count, percent}
 
       fund.applicantInfo = fund990.applicantInfo
       fund.directCharitableActivities = fund990.directCharitableActivities
@@ -212,6 +229,4 @@ module.exports = {
 
       contribution
     , {concurrency: 5}
-
-
 }

services/irs_990_importer/helpers.coffee

@@ -19,4 +19,6 @@ module.exports = {
       entityName += " #{filing.ReturnHeader.BsnssNm_BsnssNmLn2Txt}"
     entityName
 
+  roundTwoDigits: (num) ->
+    Math.round(num * 100) / 100
 }

services/irs_990_importer/index.coffee

@@ -13,13 +13,16 @@ config = require '../../config'
 # FIXME: classify community foundatoins (990 instead of 990pf) as fund and org?
 
 class Irs990Service
-  processEin: (ein) =>
-    chunk = await IrsOrg990.getAllByEin ein
+  processEin: (ein, {type}) =>
+    Model = if type is 'fund' then IrsFund990 else IrsOrg990
+    chunk = await Model.getAllByEin ein
     JobCreate.createJob {
       queue: JobService.QUEUES.DEFAULT
       waitForCompletion: true
       job: {chunk}
-      type: JobService.TYPES.DEFAULT.IRS_990_PROCESS_ORG_CHUNK
+      type: if type is 'fund' \
+            then JobService.TYPES.DEFAULT.IRS_990_PROCESS_FUND_CHUNK \
+            else JobService.TYPES.DEFAULT.IRS_990_PROCESS_ORG_CHUNK
       ttlMs: 120000
       priority: JobService.PRIORITIES.NORMAL
     }