Jugend forscht Regionalwettbewerb

.gitignore

@@ -0,0 +1,108 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# cert and key
+mycert.pem
+mykey.key
+
+# mac files
+*.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Tobias Scheithauer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,5 @@
+# DeUs
+
+Dies ist das Repository zum Jugend forscht Projekt "Warum Shakespeare wirklich Shakespeare ist - Eine Analyse künstlicher neuronaler Netze". 
+
+Fragen zum Projekt nehme ich gerne [per Mail](mailto:[email protected]) entgegen. Für Probleme mit dem Sourcecode gibt es die Issues. Das Erstellen von Forks und die Anwendung meiner Methoden auf eigene neuronale Netze ist ausdrücklich erwünscht. 

resources/1511967049/saves/checkpoint

@@ -0,0 +1,25 @@
+model_checkpoint_path: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00009384"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00000391"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00000782"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00001173"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00001564"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00001955"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00002346"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00002737"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00003128"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00003519"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00003910"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00004301"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00004692"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00005083"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00005474"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00005865"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00006256"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00006647"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00007038"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00007429"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00007820"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00008211"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00008602"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00008993"
+all_model_checkpoint_paths: "/ssd/tobias/DeepUnderstanding/src/TEXT/five-authors/runs/cnn/1511967049/saves/cnn.ckpt-00009384"

resources/README.md

@@ -0,0 +1,5 @@
+# Resources
+
+Hier finden sich die Trainingstexte, sowie die trainierten Variablen des neuronalen Netzes, welches untersucht wurde.
+
+Die Trainingstexte sind übernommen vom [Projekt Gutenberg](http://gutenberg.spiegel.de/).

resources/texts/charstats.dat

@@ -0,0 +1,12 @@
+,
+4534, 277322 goethe.txt
+26369, 1443376 kafka.txt
+12975, 510149 kleist.txt
+15713, 1149321 raabe.txt
+15354, 1076089 schiller.txt
+s
+12483, 277322 goethe.txt
+65134, 1443376 kafka.txt
+21764, 510149 kleist.txt
+50370, 1149321 raabe.txt
+48457, 1076089 schiller.txt

resources/texts/charstats.sh

@@ -0,0 +1,8 @@
+char=$1 
+echo $char >> charstats.txt
+echo `fgrep -o $char goethe.txt | wc -l`,`wc -m goethe.txt` >> charstats.txt
+echo `fgrep -o $char kafka.txt | wc -l`,`wc -m kafka.txt` >> charstats.txt
+echo `fgrep -o $char kleist.txt | wc -l`,`wc -m kleist.txt` >> charstats.txt
+echo `fgrep -o $char raabe.txt | wc -l`,`wc -m raabe.txt` >> charstats.txt
+echo `fgrep -o $char schiller.txt | wc -l`,`wc -m schiller.txt` >> charstats.txt
+

src/Generate-Activations.py

@@ -0,0 +1,94 @@
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+import matplotlib
+import model_cnn
+import preprocessing_classification as pre_c
+import itertools as it
+from scipy import signal
+import scipy
+from makedata_classification import sliding
+import glob
+from sklearn.cluster import MeanShift, estimate_bandwidth
+import time
+import sys
+
+
+sess = tf.InteractiveSession()
+
+text_length = 1000
+num_authors = 5
+input_cnn = tf.placeholder(tf.float32, [None, len(pre_c.alphabet), text_length, 1], name="input_x")
+with tf.variable_scope("cnn"):
+ cnn_logits, cnn_variables, _ = model_cnn.inference(
+ input_x=input_cnn, keep_prob=1.0, num_authors=num_authors)
+known_vars = []
+known_vars = tf.global_variables()
+[print(v) for v in known_vars]
+saver = tf.train.Saver(var_list=known_vars)
+saver.restore(sess, "../resources/1511967049/saves/cnn.ckpt-00009384")
+print("cnn_classifier restored")
+
+w_1 = sess.run('cnn/conv-maxpool-1/W:0')
+b_1 = sess.run('cnn/conv-maxpool-1/b:0')
+w_2 = sess.run('cnn/conv-maxpool-2/W:0')
+b_2 = sess.run('cnn/conv-maxpool-2/b:0')
+w_3 = sess.run('cnn/conv-3/W:0')
+b_3 = sess.run('cnn/conv-3/b:0')
+w_4 = sess.run('cnn/conv-4/W:0')
+b_4 = sess.run('cnn/conv-4/b:0')
+w_5 = sess.run('cnn/conv-5/W:0')
+b_5 = sess.run('cnn/conv-5/b:0')
+w_6 = sess.run('cnn/conv-maxpool-6/W:0')
+b_6 = sess.run('cnn/conv-maxpool-6/b:0')
+w_fc = sess.run('cnn/fc/W-fc:0')
+b_fc = sess.run('cnn/fc/b-fc:0')
+weights = [w_1, w_2, w_3, w_4, w_5, w_6, w_fc]
+biases = [b_1, b_2, b_3, b_4, b_5, b_6, b_fc]
+
+
+def prepare_cnn_input(textlist):
+ encoded_list = []
+ for sample in textlist:
+ # shorten text if it is too long
+ if len(sample) > text_length: # tf.flags.FLAGS.text_length:
+ text_end_extracted = sample.lower()[0:text_length]
+ else:
+ text_end_extracted = sample.lower()
+ # pad text with spaces if it is too short
+ num_padding = text_length - len(text_end_extracted)
+ padded = text_end_extracted + " " * num_padding
+ text_int8_repr = np.array([pre_c.alphabet.find(char) for char in padded], dtype=np.int8)
+ x_batch_one_hot = np.zeros(shape=[len(pre_c.alphabet), len(text_int8_repr), 1])
+ for char_pos_in_seq, char_seq_char_ind in enumerate(text_int8_repr):
+ if char_seq_char_ind != -1:
+ x_batch_one_hot[char_seq_char_ind][char_pos_in_seq][0] = 1
+ encoded_list.append(x_batch_one_hot)
+ return np.array(encoded_list)
+
+
+def calculate_activations(textlist):
+ logits, activations = sess.run([cnn_logits, cnn_variables], feed_dict={input_cnn: prepare_cnn_input(textlist)[:-1]})
+ logit_array = np.reshape(np.argmax(logits, axis=1), [len(logits), 1])
+ return logit_array, activations
+
+
+def data_generation(file):
+ with open(file) as f:
+ eof = False
+ ctr = 0
+ while eof == False:
+ try:
+ read = [line.split("|SEPERATOR|") for line in f.read(5000*1013).split("\n")[:-1]][0::10]
+ texts = [r[1] for r in read]
+ authors = np.expand_dims(np.array([int(r[0])-1 for r in read][:-1]), axis=1)
+ logits, act = calculate_activations(texts)
+ logits_save = np.append(logits, authors, axis=1)
+ np.savez_compressed("../resources/activations-five-authors/"+file+"-activations-"+str(ctr), logits=logits_save, act_1=act[0], act_2=act[1], act_3=act[2], act_4=act[3], act_5=act[4], act_6=act[5], act_7=act[6])
+ ctr += 1
+ except:
+ eof = True
+ print(ctr)
+
+
+data_generation(file="TrainSet-five_authors.txt")