cleanup (trailbehind#57)

hzy-zg · Jun 18, 2016 · d4fc9ae · d4fc9ae
1 parent 308bfa8
commit d4fc9ae
Show file tree

Hide file tree

Showing 12 changed files with 53 additions and 93 deletions.
diff --git a/README.md b/README.md
@@ -13,9 +13,9 @@ Running the code is as easy as install Docker, make dev, and run a script.
 
 Contributions are welcome. Open an issue if you want to discuss something to do, or [email me](mailto:[email protected]).
 
-## New Alpha Release: DeepOSM.org
+## New Alpha Release: www.DeepOSM.org
 
-[DeepOSM.org](http://www.deeposm.org/) now shows some predicted errors from DeepOSM. This is an *alpha release* - it only uses a shallow network, doesn't link up adjacent errors, and has other issues. 
+[www.DeepOSM.org](http://www.deeposm.org/) now shows some predicted errors from DeepOSM. This is an *alpha release* - it only uses a shallow network, doesn't link up adjacent errors, and has other issues. 
 
 ## Default Data/Accuracy
 

diff --git a/bin/train_neural_net.py b/bin/train_neural_net.py
@@ -8,9 +8,7 @@
 # src.training_data must be included before src.single_layer_network,
 # in order to import PIL before TFLearn - or PIL errors tryig to save a JPEG
 from src.training_data import CACHE_PATH, METADATA_PATH
-from src.s3_client_deeposm import post_findings_to_s3
 from src.single_layer_network import train_on_cached_data
-from src.training_visualization import render_errors
 
 
 def create_parser():
@@ -43,10 +41,9 @@ def main():
     test_images, model = train_on_cached_data(raster_data_paths, args.neural_net,
                                               training_info['bands'], training_info['tile_size'],
                                               args.number_of_epochs)
-    if post_findings_to_s3:
 
-    if args.render_results:
-        render_errors(raster_data_paths, predictions, test_images, band_list, tile_size)
+    with open(CACHE_PATH + 'model.pickle', 'w') as outfile:
+        pickle.dump(model, outfile)
 
 if __name__ == "__main__":
     main()
diff --git a/bin/update_deeposmorg.py b/bin/update_deeposmorg.py
@@ -9,12 +9,11 @@
 
 
 def main():
-	  """DO each state one by one."""
-    randomize_naips = False
+    """Do each state one by one."""
+    # randomize_naips = False
     naip_year = 2013
-    naip_states = ['de': ['http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf'], 
-                   'me': ['http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf']
-                  ]
+    naip_states = {'de': ['http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf'],
+                   'me': ['http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf']}
     number_of_naips = 25
 
     extract_type = 'highway'
@@ -43,7 +42,7 @@ def main():
 
     with open(CACHE_PATH + MODEL_METADATA_PATH, 'r') as infile:
         model_info = pickle.load(infile)
-
-    model = load_model(model_info['neural_net_type'], model_info['tile_size'], len(model_info['bands']))
-    post_findings_to_s3(raster_data_paths, model, training_info)
 
+    model = load_model(model_info['neural_net_type'], model_info['tile_size'],
+                       len(model_info['bands']))
+    post_findings_to_s3(raster_data_paths, model, training_info)
diff --git a/bin/upload_data.py b/bin/upload_data.py
@@ -2,7 +2,6 @@
 
 """Post test results to S3. This class is probably not relevant to you, it's for deeposm.org."""
 
-import argparse
 import pickle
 from src.s3_client_deeposm import post_findings_to_s3
 from src.single_layer_network import load_model, MODEL_METADATA_PATH
@@ -16,11 +15,12 @@ def main():
 
     with open(CACHE_PATH + METADATA_PATH, 'r') as infile:
         training_info = pickle.load(infile)
-    
+
     with open(CACHE_PATH + MODEL_METADATA_PATH, 'r') as infile:
         model_info = pickle.load(infile)
-
-    model = load_model(model_info['neural_net_type'], model_info['tile_size'], len(model_info['bands']))
+
+    model = load_model(model_info['neural_net_type'], model_info['tile_size'],
+                       len(model_info['bands']))
     post_findings_to_s3(raster_data_paths, model, training_info)
 
 

diff --git a/deeposm.org/.ebextensions/01_packages.config b/deeposm.org/.ebextensions/01_packages.config
diff --git a/deeposm.org/README.md b/deeposm.org/README.md
@@ -17,31 +17,7 @@ Then, the site is live at your docker IP, similar to: http://192.168.99.100:8000
 
     docker-compose run web /usr/local/bin/python manage.py migrate  --run-syncdb
 
-## Deploy to Amazon Elastic Beanstalk (EBS)
-
-This is based on a [Python 3 Elastic Beanstalk tutorial](https://realpython.com/blog/python/deploying-a-django-app-and-postgresql-to-aws-elastic-beanstalk/). It goes like:
-
-Use the EBS command line tools to deploy:
-
-    pip install awsebcli
-
-Per tutorial, run
-
-    eb create --database.engine postgres
-
-Everything is otherwise all configured by files in .ebextensions, and there shouldn't be anything else to do.
-
-Then:
-
-    eb deploy
-
-Also, set your AWS credentials on EB:
-
-    eb setenv AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
-    eb setenv AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-    eb setenv DJANGO_SETTINGS_MODULE=website.settings
-
-## Data Displayed
+## Creating Production Findings
 
 The production findings are created and posted to S3 by running these commands, using DeepOSM. 
 

diff --git a/deeposm.org/website/settings.py b/deeposm.org/website/settings.py
@@ -96,8 +96,8 @@
             'USER': 'postgres',
             'HOST': 'db',
             'PORT': 5432,
+        }
     }
-}
 
 
 # Password validation

diff --git a/deeposm.org/website/views.py b/deeposm.org/website/views.py
@@ -4,8 +4,6 @@
 from django.template import loader
 import boto3
 import datetime
-import json
-import operator
 import os
 import pickle
 from website import models, settings
@@ -59,10 +57,8 @@ def list_errors(request, analysis_type, country_abbrev, state_name):
 
 def sorted_findings(state_name):
     """Return a list of errors for the path, sorted by probability."""
-    return models.MapError.objects.filter(state_abbrev=STATE_NAMES_TO_ABBREVS[state_name], solved_date=None).order_by('-certainty')
-    with open(path, 'r') as infile:
-        errors = pickle.load(infile)
-    return errors
+    return models.MapError.objects.filter(state_abbrev=STATE_NAMES_TO_ABBREVS[state_name],
+                                          solved_date=None).order_by('-certainty')
 
 
 def cache_findings():
@@ -81,46 +77,48 @@ def cache_findings():
             s3_client.download_file(FINDINGS_S3_BUCKET, obj.key, local_path)
             with open(local_path, 'r') as infile:
                 errors = pickle.load(infile)
-            
+
             naip_errors = {}
             for e in errors:
                 try:
                     e['state_abbrev']
                 except:
                     break
                 filename = e['raster_filename']
-
 
                 if filename in naip_errors:
-                    # keep track of which errors dont exist for the import, to mark as solved
-                    errors_for_naip = models.MapError.objects.filter(raster_filename = filename)
+                    # keep track of which errors dont exist for the import, to
+                    # mark as solved
+                    errors_for_naip = models.MapError.objects.filter(
+                        raster_filename=filename)
                     error_ids = []
                     for err in errors_for_naip:
-                      error_ids.append(err.id)
+                        error_ids.append(err.id)
                     naip_errors[filename] = error_ids
 
                 try:
-                    map_error = models.MapError.objects.get(raster_filename = filename,
-                                                raster_tile_x = e['raster_tile_x'],
-                                                raster_tile_y = e['raster_tile_y'],
-                                                )
+                    map_error = models.MapError.objects.get(raster_filename=filename,
+                                                            raster_tile_x=e['raster_tile_x'],
+                                                            raster_tile_y=e['raster_tile_y'],
+                                                            )
                     naip_errors[filename].remove(map_error.id)
                     map_error.solved_date = None
                 except:
-                    map_error = models.MapError(raster_filename = filename,
-                                                raster_tile_x = e['raster_tile_x'],
-                                                raster_tile_y = e['raster_tile_y'],
-                                                state_abbrev = e['state_abbrev'],
-                                                ne_lat = e['ne_lat'],
-                                                ne_lon = e['ne_lon'],
-                                                sw_lat = e['sw_lat'],
-                                                sw_lon = e['sw_lon']
-                                               )
+                    map_error = models.MapError(raster_filename=filename,
+                                                raster_tile_x=e['raster_tile_x'],
+                                                raster_tile_y=e['raster_tile_y'],
+                                                state_abbrev=e['state_abbrev'],
+                                                ne_lat=e['ne_lat'],
+                                                ne_lon=e['ne_lon'],
+                                                sw_lat=e['sw_lat'],
+                                                sw_lon=e['sw_lon']
+                                                )
                 map_error.certainty = e['certainty']
                 map_error.save()
-
-            for key in naip_errors:                
-                fixed_errors = models.MapError.objects.filter(id__in=naip_errors[key])
+
+            for key in naip_errors:
+                fixed_errors = models.MapError.objects.filter(
+                    id__in=naip_errors[key])
                 for f in fixed_errors:
                     f.solved_date = datetime.date.utcnow()
                     f.save()

diff --git a/src/s3_client_deeposm.py b/src/s3_client_deeposm.py
@@ -10,7 +10,7 @@
 from src.training_visualization import render_results_for_analysis
 
 
-def post_findings_to_s3(raster_data_paths, model, training_info):
+def post_findings_to_s3(raster_data_paths, model, training_info, render_results):
     """Aggregate findings from all NAIPs into a pickled list, post to S3."""
     findings = []
     for path in raster_data_paths:
@@ -30,8 +30,7 @@ def post_findings_to_s3(raster_data_paths, model, training_info):
 
         # combine findings for all NAIP images analyzedfor the region
         [findings.append(f) for f in tag_with_locations(fp_images, false_positives,
-                                                        training_info['tile_size'], 
-                                                        training_info['naip_state'])]
+                                                        training_info['tile_size'])]
 
     # dump combined findings to disk as a pickle
     try:

diff --git a/src/single_layer_network.py b/src/single_layer_network.py
@@ -87,9 +87,9 @@ def train_with_data(onehot_training_labels, onehot_test_labels, test_images, tra
         for b in band_list:
             if b == 1:
                 on_band_count += 1
-        
+
         model = model_for_type(neural_net_type, tile_size, on_band_count)
-   
+
     model.fit(norm_train_images,
               npy_training_labels,
               n_epoch=number_of_epochs,
@@ -129,8 +129,8 @@ def save_model(model, neural_net_type, bands, tile_size):
     """Save a DeepOSM tflearn model and its metadata. """
     model.save(CACHE_PATH + 'model.pickle')
     # dump the training metadata to disk, for later loading model from disk
-    training_info = {'neural_net_type': neural_net_type, 
-                     'bands': bands, 
+    training_info = {'neural_net_type': neural_net_type,
+                     'bands': bands,
                      'tile_size': tile_size}
     with open(CACHE_PATH + MODEL_METADATA_PATH, 'w') as outfile:
         pickle.dump(training_info, outfile)

diff --git a/src/training_data.py b/src/training_data.py
@@ -418,11 +418,11 @@ def tag_with_locations(test_images, predictions, tile_size, state_abbrev):
         sw_lat, sw_lon = pixel_to_lat_lon_web_mercator(raster_dataset, raster_tile_x,
                                                        raster_tile_y + tile_size)
         certainty = predictions[idx][0]
-        formatted_info = {'certainty': certainty, 'ne_lat': ne_lat, 'ne_lon': ne_lon, 
-                          'sw_lat': sw_lat, 'sw_lon': sw_lon, 'raster_tile_x': raster_tile_x, 
-                          'raster_tile_y': raster_tile_y, 'raster_filename': raster_filename, 
+        formatted_info = {'certainty': certainty, 'ne_lat': ne_lat, 'ne_lon': ne_lon,
+                          'sw_lat': sw_lat, 'sw_lon': sw_lon, 'raster_tile_x': raster_tile_x,
+                          'raster_tile_y': raster_tile_y, 'raster_filename': raster_filename,
                           'state_abbrev': state_abbrev
-                         }
+                          }
         combined_data.append(formatted_info)
     return combined_data
 

diff --git a/src/training_visualization.py b/src/training_visualization.py
@@ -7,7 +7,7 @@
 import numpy
 from PIL import Image
 
-from src.training_data import way_bitmap_for_naip
+from src.training_data import list_findings, load_training_tiles, way_bitmap_for_naip
 
 
 def render_errors(raster_data_paths, model, training_info, render_results):