Added support for JWT tokens

stringlifier/api.py

@@ -226,6 +226,8 @@ def _extract_tokens(self, string: str, pred: NDArray[Int64]) -> Tuple[str, List[
                 type_ = '<IP_ADDR>'
             elif last_label == 'U':
                 type_ = '<UUID>'
+            elif last_label == 'J':
+                type_ = '<JWT>'
             if last_label != 'C':
                 tokens.append((c_tok, start, ii, type_))
 

stringlifier/modules/training.py

@@ -17,12 +17,13 @@
 known_words = []
 
 
+
 def _generate_word(known_words):
     import uuid
     import datetime
     import base64
     generated = None
-    ii = random.randint(0, 4)
+    ii = random.randint(0, 5)
     mask = 'H'
     if ii == 0:
         generated = str(uuid.uuid4())
@@ -61,13 +62,26 @@ def _generate_word(known_words):
             toks.append(str(random.randint(0, 255)))
         generated = '.'.join(toks)
         mask = 'I'
+    elif ii==5:
+        generated=_generate_JWT_token(known_words)
+        mask = 'J'
     return str(generated), mask[0]
 
 
 lines = open('corpus/words_alpha.txt').readlines()
 for line in lines:
     known_words.append(line.strip())
 
+
+def _generate_JWT_token(known_words):
+    import jwt
+
+    payload = {"id": str(random.random()), "client_id": str(random.random()), "user_id": str(random.random()), "type": "access_token",
+               "expires_in": str(random.randint(10,3600000)), "scope": "read, write", "created_at": str(random.randint(1900000, 9000000))}
+    encoded_jwt = jwt.encode(payload, 'secret', algorithm='HS256')
+
+    return str(encoded_jwt)[2:-1]
+
 # generated_words = generate_words(len(known_words), known_words)
 
 known_index = 0