WIP: Flake8, additional tests, updated UX (center-for-threat-informed…

…-defense#13)

* Change pytest call to cover all source files

* Add flake8 checks

* Flake8 check

* Test flake8 failure

* More flake8

* More tests

* Install punkt

* Tests, add get_training_data

* Exclude None Mappings from training data

* Big update - roughed in UX

* django-rest-framework

* unpin drf because actions cant find it (why?)

* Fix flake and test errors

* Updated UX, get_training_data

.github/workflows/test.yml

@@ -19,9 +19,10 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements/test-requirements.txt
         pip install -r requirements/requirements.txt
-    - name: Test with pytest
+        python -c "import nltk; nltk.download('punkt')"
+    - name: Test and lint with pytest
       run: |
-        pytest --cov=src/ --cov-report=xml
+        pytest --cov=src/ --cov=src/tram --cov=src/tram/tram/ml --cov=src/tram/tram/management/commands --cov-report=xml --flake8
     - name: Upload Coverage to CodeCov
       uses: codecov/codecov-action@v1
       with:

requirements/requirements.txt

@@ -1,4 +1,7 @@
+beautifulsoup4==4.9.3
 django==3.1.7
+django-rest-framework
 faker==6.5.0
 pdfplumber==0.5.27
+python-docx==0.8.10
 nltk==3.5

requirements/test-requirements.txt

@@ -2,5 +2,7 @@ bandit==1.7.0
 pytest
 pytest-cov==2.11.1
 pytest-django==4.1.0
+pytest-flake8==1.0.7
+flake8>=3.5
 pytest-pythonpath
 safety==1.10.3

setup.cfg

@@ -1,3 +1,7 @@
 [tool:pytest]
 DJANGO_SETTINGS_MODULE = tram.settings
-python_paths = src/tram
+python_paths = src/tram
+flake8-ignore = 
+    src/archive/* ALL
+    src/tram/tram/migrations/* ALL
+flake8-max-line-length = 120

src/tram/tram-schema.sql

@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS "django_migrations" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "app" varchar(255) NOT NULL, "name" varchar(255) NOT NULL, "applied" datetime NOT NULL);
+CREATE TABLE IF NOT EXISTS "auth_group_permissions" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "group_id" integer NOT NULL REFERENCES "auth_group" ("id") DEFERRABLE INITIALLY DEFERRED, "permission_id" integer NOT NULL REFERENCES "auth_permission" ("id") DEFERRABLE INITIALLY DEFERRED);
+CREATE TABLE IF NOT EXISTS "auth_user_groups" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "user_id" integer NOT NULL REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED, "group_id" integer NOT NULL REFERENCES "auth_group" ("id") DEFERRABLE INITIALLY DEFERRED);
+CREATE TABLE IF NOT EXISTS "auth_user_user_permissions" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "user_id" integer NOT NULL REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED, "permission_id" integer NOT NULL REFERENCES "auth_permission" ("id") DEFERRABLE INITIALLY DEFERRED);
+CREATE TABLE IF NOT EXISTS "django_admin_log" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "action_time" datetime NOT NULL, "object_id" text NULL, "object_repr" varchar(200) NOT NULL, "change_message" text NOT NULL, "content_type_id" integer NULL REFERENCES "django_content_type" ("id") DEFERRABLE INITIALLY DEFERRED, "user_id" integer NOT NULL REFERENCES "auth_user" ("id") DEFERRABLE INITIALLY DEFERRED, "action_flag" smallint unsigned NOT NULL CHECK ("action_flag" >= 0));
+CREATE TABLE IF NOT EXISTS "django_content_type" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "app_label" varchar(100) NOT NULL, "model" varchar(100) NOT NULL);
+CREATE TABLE IF NOT EXISTS "auth_permission" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "content_type_id" integer NOT NULL REFERENCES "django_content_type" ("id") DEFERRABLE INITIALLY DEFERRED, "codename" varchar(100) NOT NULL, "name" varchar(255) NOT NULL);
+CREATE TABLE IF NOT EXISTS "auth_group" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(150) NOT NULL UNIQUE);
+CREATE TABLE IF NOT EXISTS "auth_user" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "password" varchar(128) NOT NULL, "last_login" datetime NULL, "is_superuser" bool NOT NULL, "username" varchar(150) NOT NULL UNIQUE, "last_name" varchar(150) NOT NULL, "email" varchar(254) NOT NULL, "is_staff" bool NOT NULL, "is_active" bool NOT NULL, "date_joined" datetime NOT NULL, "first_name" varchar(150) NOT NULL);
+CREATE TABLE IF NOT EXISTS "django_session" ("session_key" varchar(40) NOT NULL PRIMARY KEY, "session_data" text NOT NULL, "expire_date" datetime NOT NULL);
+CREATE TABLE IF NOT EXISTS "tram_document" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "docfile" varchar(100) NOT NULL, "created_on" datetime NOT NULL);
+CREATE TABLE IF NOT EXISTS "tram_documentprocessingjob" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "created_on" datetime NOT NULL, "document_id" integer NOT NULL REFERENCES "tram_document" ("id") DEFERRABLE INITIALLY DEFERRED);
+CREATE TABLE IF NOT EXISTS "tram_indicator" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "report_id" integer NOT NULL REFERENCES "tram_report" ("id") DEFERRABLE INITIALLY DEFERRED, "value" varchar(200) NOT NULL, "indicator_type" varchar(200) NOT NULL);
+CREATE TABLE IF NOT EXISTS "tram_attacktechnique" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(200) NOT NULL, "stix_id" varchar(128) NOT NULL UNIQUE, "attack_id" varchar(128) NOT NULL UNIQUE, "attack_url" varchar(512) NOT NULL, "matrix" varchar(200) NOT NULL);
+CREATE TABLE IF NOT EXISTS "tram_report" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(200) NOT NULL, "created_on" datetime NOT NULL, "updated_on" datetime NOT NULL, "document_id" integer NOT NULL REFERENCES "tram_document" ("id") DEFERRABLE INITIALLY DEFERRED, "ml_model" varchar(200) NOT NULL, "text" text NOT NULL);
+CREATE TABLE IF NOT EXISTS "tram_mapping" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "confidence" real NOT NULL, "disposition" varchar(200) NOT NULL, "attack_technique_id" integer NULL REFERENCES "tram_attacktechnique" ("id") DEFERRABLE INITIALLY DEFERRED, "report_id" integer NOT NULL REFERENCES "tram_report" ("id") DEFERRABLE INITIALLY DEFERRED, "sentence_id" integer NOT NULL REFERENCES "tram_sentence" ("id") DEFERRABLE INITIALLY DEFERRED);
+CREATE TABLE IF NOT EXISTS "tram_sentence" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "text" text NOT NULL, "document_id" integer NOT NULL REFERENCES "tram_document" ("id") DEFERRABLE INITIALLY DEFERRED, "order" integer NOT NULL);
+CREATE UNIQUE INDEX "auth_group_permissions_group_id_permission_id_0cd325b0_uniq" ON "auth_group_permissions" ("group_id", "permission_id");
+CREATE INDEX "auth_group_permissions_group_id_b120cbf9" ON "auth_group_permissions" ("group_id");
+CREATE INDEX "auth_group_permissions_permission_id_84c5c92e" ON "auth_group_permissions" ("permission_id");
+CREATE UNIQUE INDEX "auth_user_groups_user_id_group_id_94350c0c_uniq" ON "auth_user_groups" ("user_id", "group_id");
+CREATE INDEX "auth_user_groups_user_id_6a12ed8b" ON "auth_user_groups" ("user_id");
+CREATE INDEX "auth_user_groups_group_id_97559544" ON "auth_user_groups" ("group_id");
+CREATE UNIQUE INDEX "auth_user_user_permissions_user_id_permission_id_14a6b632_uniq" ON "auth_user_user_permissions" ("user_id", "permission_id");
+CREATE INDEX "auth_user_user_permissions_user_id_a95ead1b" ON "auth_user_user_permissions" ("user_id");
+CREATE INDEX "auth_user_user_permissions_permission_id_1fbb5f2c" ON "auth_user_user_permissions" ("permission_id");
+CREATE INDEX "django_admin_log_content_type_id_c4bce8eb" ON "django_admin_log" ("content_type_id");
+CREATE INDEX "django_admin_log_user_id_c564eba6" ON "django_admin_log" ("user_id");
+CREATE UNIQUE INDEX "django_content_type_app_label_model_76bd3d3b_uniq" ON "django_content_type" ("app_label", "model");
+CREATE UNIQUE INDEX "auth_permission_content_type_id_codename_01ab375a_uniq" ON "auth_permission" ("content_type_id", "codename");
+CREATE INDEX "auth_permission_content_type_id_2f476e4b" ON "auth_permission" ("content_type_id");
+CREATE INDEX "django_session_expire_date_a5c62663" ON "django_session" ("expire_date");
+CREATE INDEX "tram_documentprocessingjob_document_id_2f46e4ec" ON "tram_documentprocessingjob" ("document_id");
+CREATE INDEX "tram_indicator_report_id_61f010c7" ON "tram_indicator" ("report_id");
+CREATE INDEX "tram_report_document_id_adda736a" ON "tram_report" ("document_id");
+CREATE INDEX "tram_mapping_attack_technique_id_88a7e6b3" ON "tram_mapping" ("attack_technique_id");
+CREATE INDEX "tram_mapping_report_id_249f6831" ON "tram_mapping" ("report_id");
+CREATE INDEX "tram_mapping_sentence_id_0d715d40" ON "tram_mapping" ("sentence_id");
+CREATE INDEX "tram_sentence_document_id_19970a69" ON "tram_sentence" ("document_id");