feat(tiger_downloader): allow filtering by county code, improve confi…

…g parsing logic and handling of zero padding for integer values

readme.md

@@ -313,7 +313,11 @@ the `PELIAS_CONFIG` environment variable should be set. You can read more detail
 Note that `datapath` will default to `./data/downloads` if not specified.
 
 To filter the TIGER data download you can set `state_code` property in the `pelias-config` file to the 2 digit code of the state to be downloaded.
+
+Note: some state codes begin with a leading zero, you may specify a string value or omit the extra zero and provide an integer value.
+
 In the example configuration above, the state code for Oregon, `41`, is used to limit the download.
+
 The state code can found by referencing the table below. If no `state_code` value is found, all US data will be downloaded.
 
 | code | state |
@@ -371,6 +375,24 @@ The state code can found by referencing the table below. If no `state_code` valu
 | 55 | Wisconsin            |
 | 56 | Wyoming              |
 
+For more fine-grained control, you can also set the `county_code` property in the `pelias-config` file to the 3 digit code of the state to be downloaded.
+
+Note: some county codes begin with a leading zero, you may specify a string value or omit the extra zero and provide an integer value.
+
+Note: you must specify a 'state_code' when specifying a 'county_code'.
+
+```
+"states": [
+  {
+    "state_code": 41, county_code: 1
+  }
+]
+```
+
+Check [the census website](https://www.census.gov/geographies/reference-files/2016/demo/popest/2016-fips.html) for a complete list of state and county FIPS codes.
+
+### docker example
+
 ```bash
 # prepare a build directory and a data directory to hold the newly created database files
 mkdir -p /tmp/data/berlin

script/js/update_tiger.js

@@ -12,7 +12,14 @@ let TARGET_DIR = _.get(config, 'imports.interpolation.download.tiger.datapath',
 let STATES = _.get(config, 'imports.interpolation.download.tiger.states', []);
 
 if (_.isEmpty(STATES)) {
-  STATES = [ {state_code: '' } ];
+  STATES = [ {state_code: '', county_code: ''} ];
+}
+
+// paddedInt - take an integer/string, return zero padded string
+function paddedInt(num, size) {
+  let s = num + '';
+  while (s.length < size){ s = '0' + s; }
+  return s;
 }
 
 // iterate over all the desired states, or get all if no states specified
@@ -30,7 +37,8 @@ function download(state, callback) {
     ftp: new JSFtp({
       host: 'ftp2.census.gov'
     }),
-    stateCode: state.state_code || '',
+    stateCode: state.hasOwnProperty('state_code') ? parseInt(state.state_code, 10) : '',
+    countyCode: state.hasOwnProperty('county_code') ? parseInt(state.county_code, 10) : '',
     files: []
   };
 
@@ -51,7 +59,19 @@ function download(state, callback) {
 
 function getFilteredFileList(context, callback) {
   // note that if context.stateCode is an empty string, all files will be listed
-  context.ftp.list(`/geo/tiger/TIGER2016/ADDRFEAT/tl_2016_${context.stateCode}*.zip`, (err, res) => {
+  let filter = '';
+  if( context.stateCode ){
+    filter += paddedInt( context.stateCode, 2 );
+    if( context.countyCode ){
+      filter += paddedInt( context.countyCode, 3 );
+    }
+  } else {
+    if( context.countyCode ){
+      logger.error(`county_code specified but state_code is invalid`);
+      return callback();
+    }
+  }
+  context.ftp.list(`/geo/tiger/TIGER2016/ADDRFEAT/tl_2016_${filter}*.zip`, (err, res) => {
     if (err) {
       return callback(err);
     }