From 1966b291ed5acbd6ddb01fbb261847637d133b0d Mon Sep 17 00:00:00 2001 From: Luc Bertin Date: Mon, 14 Dec 2020 12:32:37 +0100 Subject: [PATCH] commit-- --- .gitignore | 14 +++ READMe.md | 55 +++++++++++ app.py | 191 +++++++++++++++++++++++++++++++++++++ config.py | 38 ++++++++ model_saved | Bin 0 -> 818 bytes requirements.txt | 27 ++++++ templates/404.html | 11 +++ templates/base.html | 27 ++++++ templates/index.html | 10 ++ templates/navigation.html | 30 ++++++ templates/predict.html | 22 +++++ templates/results.html | 38 ++++++++ templates/users.html | 34 +++++++ tests/__init__.py | 0 tests/test_app.py | 22 +++++ tests/test_integration.py | 60 ++++++++++++ tests/test_requirements.py | 27 ++++++ tests/utils.py | 33 +++++++ 18 files changed, 639 insertions(+) create mode 100644 .gitignore create mode 100644 READMe.md create mode 100644 app.py create mode 100644 config.py create mode 100644 model_saved create mode 100644 requirements.txt create mode 100644 templates/404.html create mode 100644 templates/base.html create mode 100644 templates/index.html create mode 100644 templates/navigation.html create mode 100644 templates/predict.html create mode 100644 templates/results.html create mode 100644 templates/users.html create mode 100644 tests/__init__.py create mode 100644 tests/test_app.py create mode 100644 tests/test_integration.py create mode 100644 tests/test_requirements.py create mode 100644 tests/utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49feeac --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# ignore all Python compiled file +*.pyc + +# ignore the virtual env (the user will create it and install packages from requirements.txt) +myenv/ + +# ignore all __pycache__ folders +__pycache__ + +# from Mac +.DS_Store + +# from pyenv +.python-version \ No newline at end of file diff --git a/READMe.md b/READMe.md new file mode 100644 index 0000000..cf2b8c8 --- /dev/null +++ b/READMe.md @@ -0,0 +1,55 @@ +# Last session project. +## Duration: 3h + +# Brief description + +This is an overly simplistic application developped using the Flask micro web-framework. +Of course you can do better in terms of code refactoring (especially the app.py file), but that's not the main focus here. +It uses an already trained (logistic regression) model to predict if the user would have had survived to the Titanic, given the data he submits to the server. +It also caches the predictions and inputs in a Postgres database so that a user may only submit its details once (can't retry a prediction with the same username). + +What we ask you to do is: +1. **containerize this app**, bundling it along with its dependencies using a **Dockerfile** +2. **orchestrate it** using a **docker-compose.yml file**, with a **Postgres server** using the corresponding **Docker image** from the Docker Hub registry. + +# Prerequities + +1. You **must** have a GitHub account. +2. **Fork** this repository. +3. Git **clone** the **forked** repository into a local repository: `git clone your_forked_repo_https_url` +4. PLay with the app, read the code +5. Create a **Dockerfile + docker-compose.yml** file +6. Check if your containerized app works\* +7. Git push\* +8. Post your name, lastname (as appears on the DVO/attendance list) and Github link here: https://forms.gle/qhdQQ2mcgPenCgdb7 + +# Must do ! +- Dockerfile and docker-compose.yml file should be in **root project folder**. +- the service name for the flask application in the docker-compose file **must** be named ***flaskapp***. +- I must be able to run your project only by doing `docker-compose up --build`, no more! +- In the dockerfile, you must copy the whole content of the flaskapp folder in a dedicated folder in the container. +- The Python code **MUST NOT** be modified, **NOR** moved under **NO** circumstances, **only 1 Dockerfile and 1 docker-compose file should be created** + +Anything that does not respect the aforementioned conditions will result in an automatic score of 0 for this exam. This is CRUCIAL so that I can perform the tests on your project. + +\* How to push code, +Do your modifications, and at the end of the 3h session, from your **root project folder**, do from your terminal: +```sh + git add . + git commit -m "first commit" + git push origin master +``` + +To install git on your [computer](https://git-scm.com/book/fr/v2/Démarrage-rapide-Installation-de-Git) +in [english](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). + +## Tips + +1. Include this block in your Dockerfile +``` + RUN wget https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -P /scripts + RUN chmod +x /scripts/wait-for-it.sh + ENTRYPOINT ["/scripts/wait-for-it.sh", "db:5432", "--"] +``` +2. Running the app can be done simply `python app.py runserver --host=0.0.0.0 --threaded`. This should then also be the **Dockerfile default startup command**. +3. Look at the whole project (code, config file, etc.) to check the dependencies to install, name of the service to give, and env variables to be set. \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..5a47e1d --- /dev/null +++ b/app.py @@ -0,0 +1,191 @@ +## +## flask imports +## +from flask import Flask, current_app +from flask import (make_response, + redirect, + abort, + session, + render_template, + url_for, + flash, + request) +## +## Flask extensions imports +## +from flask_script import Manager +from flask_bootstrap import Bootstrap +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate, MigrateCommand +from flask_wtf import FlaskForm +from wtforms import (StringField, + SubmitField, IntegerField, FloatField, SelectField) +from wtforms.validators import (DataRequired, + InputRequired, Length, NumberRange) +## +## configuration file imports +## +import config + + +## +## creation of the Flask application instance +## +## passing its configuration, and initializing the extensions along with the flask app. +## The web server passes all requests it receives from clients to this object for handling, using a protocol called Web Server Gateway Interface (WSGI) +app = Flask(__name__, template_folder="templates") +app.config.from_object('config') + +manager = Manager(app) # To give provide CLI commands +bootstrap = Bootstrap(app) # Twitter Bootstrap extension +db = SQLAlchemy(app) # SQLAlchemy extension, manipulating the database object db +migrate = Migrate(app, db) # database migrations +manager.add_command('db', MigrateCommand) # to add it as command line using Flask Script + + +## +## Forms +## + +class EnterYourInfosForm(FlaskForm): + """ + This field is the base for most of the more complicated fields, and represents an . + """ + name = StringField("Name", validators=[DataRequired()]) + + """ + Represents an + """ + age = IntegerField("Age", + validators=[DataRequired(), NumberRange(min=0, max=123)]) + """ + A text field, except all input is coerced to an float. Erroneous input is ignored and will not be accepted as a value + """ + ticket_price = FloatField("Ticket Price", + validators=[DataRequired(), NumberRange(min=0)]) + """ + Select fields take a choices parameter which is a list of (value, label) pairs. It can also be a list of only values, in which case the value is used as the label. The value can be any type, but because form data is sent to the browser as strings, you will need to provide a coerce function that converts a string back to the expected type. + """ + sexe = SelectField("Sexe", + choices=[(0, "Homme"),(1,"Femme")], + validators=[InputRequired()], + coerce=int) + """ + Represents an . This allows checking if a given submit button has been pressed. + """ + submit = SubmitField("Submit") + + #def clear(self): + #self.name.data = '' + #self.age.raw_data = [''] + #self. + + +## +## models +## + +class User(db.Model): + """ + This class describes an user. + SQLAlchemy extension for flask provides an ORM (Object-relational mappers). + An ORM is a tool that provides convertion from high-level (object-oriented) objects into low-level database instructions, so you don't have to directly deal with tables, documents, or query languages. + + for SQLAlchemy the abstraction is even higher as you can choose different database engines that relies on the same Python object (hence, without having to rewrite or adapt code)! + """ + __tablename__ = "users" # renaming the table and not default given name "user" not respecting the conventions + + id_ = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(64), unique=True) + age = db.Column(db.Integer, nullable=False) + ticket_price = db.Column(db.Float, nullable=False) + sexe = db.Column(db.Boolean, nullable=False) + survived = db.Column(db.Boolean, nullable=True) + #date_submitted = db.Column(db.DateTime, nullable=False) + + def __repr__(self): + return "User {}, with name: {}".format(self.id_, self.name) + +## +## errors handlers +## def: is a function that returns a response when a type of error is raised. +## here only one is implemented for the 404 Not Found +## The good old “chap, you made a mistake typing that URL” message. +## + +@app.errorhandler(404) +def page_not_found(e): + return render_template('404.html', nompath=e), 404 + +## +## views +## + +@app.route('/') +def index(): + return render_template("index.html") + +@app.route('/predict', methods=['GET', 'POST']) +def predict(): + form = EnterYourInfosForm() + if request.method=="POST" and form.validate_on_submit(): + # check if a user with the same name already exist in the db + user = User.query.filter_by(name=form.name.data).first() + if user is not None: + flash('It seems you already played, this was your prediction: {}'.format( + user.survived)) + else: + import joblib + loaded_model = joblib.load("model_saved") + # predict for the user if he/she would have had survived + prediction = bool(loaded_model.predict( + [[form.age.data, form.ticket_price.data, form.sexe.data]] + )) + + # create a new "user" of the service + user = User(name=form.name.data, + age=form.age.data, + ticket_price=form.ticket_price.data, + sexe=form.sexe.data, + survived=prediction) + # add the user to the db + db.session.add(user) + db.session.commit() + session["survived"] = user.survived + return redirect(url_for('show_result')) + return render_template('predict.html', form=form) + + +@app.route('/results') +def show_result(): + return render_template('results.html', success=session.get('survived', 'first')) + +@app.route('/users') +def show_all_users(): + users = User.query.all() + return render_template('users.html', users=users) + +@app.route('/') +def error_test(nompath): + # raises an HTTPException of status code 404 + # nompath is the payload + # retrieved from e in the page_not_found(e) error_handler + abort(404, nompath) + + +@manager.command +def test(): + """ Run the integration tests """ + import unittest + tests = unittest.TestLoader().discover('tests') + unittest.TextTestRunner(verbosity=2).run(tests) + +if __name__ == "__main__": + # app.run(debug=True), + # changed to manager + # using the extension + # so you can pass options + # directly from the command-line + with app.app_context(): + # create all the tables + db.create_all() + manager.run() diff --git a/config.py b/config.py new file mode 100644 index 0000000..995c68d --- /dev/null +++ b/config.py @@ -0,0 +1,38 @@ +""" +This module defines the environment variables that will be used to configure the flask application + +basedir: absolute path to the directory including this module and the app module + +SECRET_KEY: the secret key is accessed through an environment variable instead of being embedded in the code.This variable is used as a general-purpose encryption key by Flask and several third-party extensions. Hence choose a very hard string to remember to replace the default 'hard to guess string' + +SQLALCHEMY_DATABASE_URI: database is specified as a URI. +i.e. +dbengine://username:password@hostname/database +- dbengine is the database engine to use among mysql, postgresql or sqlite. +- hostname refers to the server that hosts the MySQL service +- username and password are used for authentification +- database refers to the name of the database + +SQLite databases do not have a server, but are simple files, so hostname, username, and password are omitted and database is the filename of a disk file: e.g. +'sqlite:///' + os.path.join(basedir, 'data.sqlite')' +""" + +import os + + +basedir = os.path.abspath(os.path.dirname(__file__)) + +SECRET_KEY = os.getenv("SECRET_KEY") or "hard to guess string" + +# To use a postgres db hosted on a postgres server, set the following env vars and USE_POSTGRES to any non-falsy values (e.g. 1) +password = os.getenv("POSTGRES_PASSWORD") +username = os.getenv("POSTGRES_USER") +database = os.getenv("POSTGRES_DB") +hostname = "db" +if os.getenv("USE_POSTGRES"): + # use a psotgres db using above env variables + SQLALCHEMY_DATABASE_URI = f"postgresql://{username}:{password}@{hostname}/{database}" +else: + # simply use a SQLlite database (<=> flat file) + SQLALCHEMY_DATABASE_URI = os.getenv("SQLALCHEMY_DATABASE_URI") or \ + 'sqlite:///' + os.path.join(basedir, 'data.sqlite') diff --git a/model_saved b/model_saved new file mode 100644 index 0000000000000000000000000000000000000000..e114ff7016e7b5824f2d9b46fe78f11497c14950 GIT binary patch literal 818 zcmZ{j&ubGw6vwkk)1+w?tDaO^e_}*c8O;(` ziUj{8Rw0uvkt!0}hyd*2JEghFqKs)RR040|!gF{EE4Gocgi132Ta|6FGht38ytXx? zB;4v8yY}__iH%k3Y@qX_6g+Qq$0pR zf?e$Hy6ag+2lq3Yrie3XR#WGGNmMVg5Ge`iZY&1by5=-RxatsXm4%qf=WIPR1Kj1y zV+$jz;yRweb6&wFtRR(h1G?Sr?ccbHk5@6^+ZL5l^+_Dl6t;4IulT-M9POi@Rk&p+|FKh+gIH- +
+

{{ nompath }} is not found !

+
+ +{% endblock %} diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..fd5701d --- /dev/null +++ b/templates/base.html @@ -0,0 +1,27 @@ +{# The html comment tag does not work in a jinja template but this does #} + +{# extends declares that this template inherits from bootstrap/base template where blocks'content can be overriden #} +{% extends "bootstrap/base.html" %} + +{# example of block to (re)define, check Bootstrap template #} +{% block title %}My Flask App{% endblock %} + +{% block navbar %} + {% include 'navigation.html' %} +{% endblock %} + +{% block content %} +
+ {# for the messages that get flashed #} + {% for message in get_flashed_messages() %} +
+ + {{ message }} +
+ {% endfor %} + + {# Here i define a brand new block and a child template inheriting from this template will later fill it by redeclaring it #} + {% block page_content %} + {% endblock %} +
+{% endblock %} \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..dbd8091 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} + +{% block page_content %} + +
+

Welcome to Titanic Prediction App !

+

This is a web app where you can assess whether you would have survived based on your age, the ticket price you would have invested, and whether you are a female or a male

+
+ Play +{% endblock %} \ No newline at end of file diff --git a/templates/navigation.html b/templates/navigation.html new file mode 100644 index 0000000..74baeb0 --- /dev/null +++ b/templates/navigation.html @@ -0,0 +1,30 @@ +{# Highlighting Active Menu Item +Often you want to have a navigation bar with an active navigation item. This is really simple to achieve. Because assignments outside of blocks in child templates are global and executed before the layout template is evaluated it’s possible to define the active menu item in the child template #} + +{# path/href, id, caption #} +{% set navigation_bar = [ + ('/', 'index', 'Welcome'), + ('/predict', 'predict', 'Predict'), + ('/results', 'results', 'Results'), + ('/users', 'users', 'Users') +] -%} + +{% set active_page = active_page|default('index') -%} + + + + diff --git a/templates/predict.html b/templates/predict.html new file mode 100644 index 0000000..c67bc36 --- /dev/null +++ b/templates/predict.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} + +{% set active_page = 'predict' -%} + +{# you can create your own 'functions' also named as 'macros' #} +{# to create a macro: + - you can use the {% macro function(param) %} block definition and later use it within a {{}} + - or you can define multiple macros in a dedicated template .html file then import it within this template using the {% import ..... as alias %} block definition +#} + +{% import "bootstrap/wtf.html" as wtf %} + +{% block page_content %} + +
+

Fill in the form and see if you survive !

+
+ +{# wtf.quick_form() to render the form object using default Bootstrap styles #} +{{ wtf.quick_form(form) }} + +{% endblock %} \ No newline at end of file diff --git a/templates/results.html b/templates/results.html new file mode 100644 index 0000000..0332fb2 --- /dev/null +++ b/templates/results.html @@ -0,0 +1,38 @@ +{% extends "base.html" %} + +{% set active_page = 'results' -%} + +{# you can create your own 'functions' also named as 'macros' #} +{# to create a macro: + - you can use the {% macro function(param) %} block definition and later use it within a {{}} + - or you can define multiple macros in a dedicated template .html file then import it within this template using the {% import ..... as alias %} block definition +#} + + +{% block page_content %} + + +{% if success == 'first' %} + +{% elif success %} + +{% else %} + +{% endif %} + +{% endblock %} \ No newline at end of file diff --git a/templates/users.html b/templates/users.html new file mode 100644 index 0000000..ee8be82 --- /dev/null +++ b/templates/users.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} + +{% set active_page = 'users' -%} + +{% block page_content %} + + + + + + + + + + + + + + + {% for user in users %} + + + + + + + + + {% endfor %} + +
idnameagesexeticket pricesurvived?
{{ user.id_ }}{{ user.name }}{{ user.age }}{{ user.sexe }}{{ user.ticket_price }}{{ user.survived }}
+ + +{% endblock %} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..74a28f1 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,22 @@ +import unittest +from flask import current_app +from app import app, db +import os + +class TestOnApp(unittest.TestCase): + """Check some app configuration""" + def setUp(self): + self.app = app + self.app_context = self.app.app_context() + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + + def test_app_exists(self): + self.assertIsNotNone(current_app) + + def test_valid_secret_key(self): + """assert whether SECRET_KEY env variable is set to something different from the default unsafe value""" + self.assertNotEqual(current_app.config["SECRET_KEY"], + "hard to guess string") \ No newline at end of file diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..4b124ac --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,60 @@ +import unittest +from app import app, db, User +import os +from utils import ping, connect_to_db_server + +class TestsOnEnvVars(unittest.TestCase): + + def test_postgres_env_vars_are_set(self): + """assert whether postgres-related env variables are defined""" + self.assertIsNotNone(os.getenv("POSTGRES_PASSWORD")) + self.assertIsNotNone(os.getenv("POSTGRES_USER")) + self.assertIsNotNone(os.getenv("POSTGRES_DB")) + + def test_use_postgres_env_var_is_set(self): + """assert whether use of postgres related env variable is defined""" + self.assertIsNotNone(os.getenv("USE_POSTGRES")) + +class TestIsPostgresReachable(unittest.TestCase): + """Check the Postgres service is reachable""" + + def setUp(self): + self.pguser = os.getenv("POSTGRES_USER") + self.pgpass = os.getenv("POSTGRES_PASSWORD") + self.pgdb = os.getenv("POSTGRES_DB") + + def test_is_postgres_service_reachable(self): + """Check the Postgres service is reachable by pinging it""" + self.assertTrue(ping("db")) + + def test_connect_to_postgres_db(self): + """Check if the app can connect to the postgres db hosted on the postgres server using the right credentials set on both ends""" + self.connection = connect_to_db_server( + self.pguser, self.pgpass, self.pgdb) + self.assertIsNotNone(self.connection) + +#class LoadUserInPostgresDbUseCase(unittest.TestCase): +# """ Check the Postgres service is reachable """ +# def setUp(self): +# self.app = app +# self.app_context = self.app.app_context() +# self.app.testing = True +# self.app_context.push() +# db.create_all() +# +# def test_load_user(self): +# """ Check the Postgres service is reachable by pinging it""" +# #self.assertTrue(ping("db")) +# print( self.app.config) +# with self.app.test_client() as client: +# print( self.app.config) +# client.post("/predict", +# data=dict(name="Luc", age=25, +# ticket_price=100, sexe=True)) +# self.assertIsNotNone(User.query.first()) +# +# def tearDown(self): +# db.session.remove() +# db.drop_all() +# self.app_context.pop() + diff --git a/tests/test_requirements.py b/tests/test_requirements.py new file mode 100644 index 0000000..37dd4bb --- /dev/null +++ b/tests/test_requirements.py @@ -0,0 +1,27 @@ +"""Test availability of required packages.""" + +import unittest +from pathlib import Path + +import pkg_resources + +_REQUIREMENTS_PATH = Path(__file__).parent.with_name("requirements.txt") + + +class TestRequirements(unittest.TestCase): + """Test availability of required packages.""" + + def test_requirements(self): + """Test that each required package is available.""" + # Ref: https://stackoverflow.com/a/45474387/ + requirements = pkg_resources.parse_requirements(_REQUIREMENTS_PATH.open()) + for requirement in requirements: + requirement = str(requirement).split('==')[0] # not even looking at the version + # With subtests you can know exactly the full subset of requirements that failed, so it's frequently useful while testing. Without subtests, you only know the first failure. – Acumenus + # https://stackoverflow.com/questions/16294819/check-if-my-python-has-all-required-packages + #with self.subTest(requirement=requirement): + # pkg_resources.require(requirement) + + # here, if it fails i will not look any further + pkg_resources.require(requirement) + diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..b919fb3 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,33 @@ +def ping(host): + """ + Returns True if host (str) responds to a ping request. + Remember that a host may not respond to a ping (ICMP) request even if the host name is valid. + """ + import platform # For getting the operating system name + import subprocess # For executing a shell command + + # Option for the number of packets as a function of + param = '-n' if platform.system().lower()=='windows' else '-c' + + # Building the command. Ex: "ping -c 1 google.com" + command = ['ping', param, '1', host] + + return subprocess.call(command) == 0 + +def connect_to_db_server(user, password, + database, host="db", port="5432"): + """ Connect to database server with provided environment variables """ + from psycopg2 import connect + try: + connection = connect( + user=user, + password=password, + database=database, + host=host, + port=port) + cursor = connection.cursor() + #print("Successfully connected to Postgres Server\n") + return connection + except Exception as e: + #print(f"could not connect to the postgres {e}\n") + return None \ No newline at end of file