browsers: switch to websocket, push notification of state update rath…

…er than periodic ping

cleanup: cleanup immediately when ping conn closed, allow fixed session duration
base_browser: move run_browser script to be mounted to allow easier modifications
pywb redis: use multicommand to update redis
ui: add browser about

app/browser_app.py

@@ -1,5 +1,6 @@
-#from selenium import webdriver
-#from selenium.webdriver.common.proxy import *
+from gevent import monkey, spawn, Timeout, sleep
+monkey.patch_all()
+
 
 from bottle import route, default_app, run, request, response, redirect
 
@@ -10,9 +11,16 @@
 import time
 import sys
 import os
+import json
+import traceback
 
 from argparse import ArgumentParser
 
+from bottle.ext.websocket import GeventWebSocketServer
+from bottle.ext.websocket import websocket
+
+from geventwebsocket.exceptions import WebSocketError
+
 
 PYWB_HOST_PORT = os.environ.get('PYWB_HOST_PORT', 'netcapsule_pywb_1:8080')
 
@@ -24,7 +32,8 @@
 
 pywb_ip = None
 start_url = None
-start_ts = None
+
+curr_ts = None
 
 redis = None
 local_redis = None
@@ -42,6 +51,9 @@ def set_timestamp(timestamp):
         r = requests.get('http://set.pywb.proxy/', params=params, proxies={'http': PYWB_HOST_PORT, 'https': PYWB_HOST_PORT})
 
         if r.status_code == 200:
+            global curr_ts
+            curr_ts = timestamp
+
             return {'success': r.json()}
         else:
             return {'error': r.body}
@@ -50,28 +62,75 @@ def set_timestamp(timestamp):
         return {'error': str(e)}
 
 
-@route('/set')
-def route_set_ts():
-    ts = request.query.get('ts')
-    res = set_timestamp(ts)
-    return res
+#@route('/set')
+#def route_set_ts():
+#    ts = request.query.get('ts')
+#    res = set_timestamp(ts)
+#    return res
+
+@route('/pingsock', apply=[websocket])
+def pingsock(ws):
+    spawn(receiver, ws)
+
+    last_data = None
+    sleep_timeout = 0.5
+
+    while True:
+        try:
+            data = get_update()
+            if data != last_data:
+                logging.debug('Sending' + str(data))
+                ws.send(json.dumps(data))
+                last_data = data
+        except WebSocketError as e:
+            traceback.print_exc(e)
+            mark_for_removal()
+            break
+        except Exception as e:
+            traceback.print_exc(e)
+
+        sleep(sleep_timeout)
+
+def receiver(ws):
+    while True:
+        data = ws.receive()
+        logging.debug('Received' + str(data))
+        if data is None:
+            continue
+
+        try:
+            data = json.loads(data)
+            if data['ts']:
+                set_timestamp(data['ts'])
+
+        except WebSocketError as e:
+            break
+
+        except Exception as e:
+            traceback.print_exc(e)
+
+def mark_for_removal():
+    redis.delete('c:' + HOST)
+
+    keylist = redis.keys(my_ip + ':*')
+    for key in keylist:
+        local_redis.delete(key)
 
 
-@route(['/ping'])
-def ping():
-    if not redis.hget('all_containers', HOST):
-        return
+def get_update():
+#    if not redis.hget('all_containers', HOST):
+#        return
 
-    global expire_time
-    expire_time = redis.get('container_expire_time')
-    if not expire_time:
-        expire_time = DEF_EXPIRE_TIME
+#    global expire_time
+#    expire_time = redis.get('container_expire_time')
+#    if not expire_time:
+#        expire_time = DEF_EXPIRE_TIME
 
-    redis.expire('c:' + HOST, expire_time)
+#    redis.expire('c:' + HOST, expire_time)
 
-    ts = request.query.get('ts')
+    #ts = request.query.get('ts')
 
-    base_key = my_ip + ':' + ts + ':'
+    base_key = my_ip + ':' + curr_ts + ':'
 
     # all urls
     all_urls = local_redis.hgetall(base_key + 'urls')
@@ -88,11 +147,12 @@ def ping():
     # all_hosts
     all_hosts = local_redis.smembers(base_key + 'hosts')
 
-    referrer = local_redis.get(base_key + 'r')
+    referrer = local_redis.get(base_key + 'ref')
 
     referrer_secs = int(all_urls.get(referrer, 0))
 
     return {'urls': count,
+            'req_ts': curr_ts,
             'min_sec': min_sec,
             'max_sec': max_sec,
             'hosts': list(all_hosts),
@@ -150,8 +210,8 @@ def do_init():
         start_url = 'http://' + start_url
 
     # not used here for now
-    global start_ts
-    start_ts = r.start_ts
+    global curr_ts
+    curr_ts = r.start_ts
 
     global redis
     redis = StrictRedis(REDIS_HOST)
@@ -163,8 +223,8 @@ def do_init():
         local_redis = redis
 
     # set initial url
-    base_key = my_ip + ':' + start_ts + ':'
-    local_redis.set(base_key + 'r', start_url)
+    #base_key = my_ip + ':' + curr_ts + ':'
+    #local_redis.set(base_key + 'r', start_url)
 
     return default_app()
 
@@ -178,5 +238,5 @@ def enable_cors():
 
 
 if __name__ == "__main__":
-    run(host='0.0.0.0', port='6082')
+    run(host='0.0.0.0', port='6082', server=GeventWebSocketServer)
 

app/config.yaml

@@ -2,7 +2,8 @@ api_version: '1.21'
 vnc_port: 6080
 cmd_port: 6082
 
-container_expire_secs: 90
+container_expire_secs: 600
+
 queue_expire_secs: 30
 remove_expired_secs: 20
 
@@ -19,6 +20,7 @@ browsers:
       version: '40'
       icon: 'firefox.png'
       path: firefox
+      about: 'https://en.wikipedia.org/wiki/Firefox'
 
     # chrome
     - id: chrome
@@ -27,6 +29,7 @@ browsers:
       version: '46'
       icon: 'chrome.png'
       path: chrome
+      about: 'https://en.wikipedia.org/wiki/Google_Chrome'
 
     # mosaic
     - id: mosaic
@@ -35,6 +38,7 @@ browsers:
       version: '2.2'
       icon: 'mosaic.png'
       path: mosaic
+      about: 'https://en.wikipedia.org/wiki/Mosaic_%28web_browser%29'
 
     # netscape 
     - id: netscape
@@ -43,34 +47,39 @@ browsers:
       version: '4.79'
       icon: 'win32-ns4.png'
       path: nslinux
+      about: 'https://en.wikipedia.org/wiki/Netscape_%28web_browser%29'
 
     - id: netscape4.8-win
       name: 'Netscape Navigator'
       os: 'Windows'
       version: '4.8'
       icon: 'win32-ns4.png'
       path: nswin
+      about: 'https://en.wikipedia.org/wiki/Netscape_%28web_browser%29'
 
     - id: netscape-mac-3.04
       name: 'Netscape Navigator'
       os: 'Macintosh'
       version: '3.04'
       icon: 'mac-ns2-3.png'
       path: nsmac3
+      about: 'https://en.wikipedia.org/wiki/Netscape_%28web_browser%29'
 
     - id: netscape-mac-4.08
       name: 'Netscape Navigator'
       os: 'Macintosh'
       version: '4.08'
       icon: 'mac-ns4.png'
       path: nsmac4
+      about: 'https://en.wikipedia.org/wiki/Netscape_%28web_browser%29'
 
     - id: netscape4.8-mac
       name: 'Netscape Navigator'
       os: 'Macintosh'
       version: '4.8'
       icon: 'mac-ns4.png'
       path: nsmac4.8
+      about: 'https://en.wikipedia.org/wiki/Netscape_%28web_browser%29'
 
     # Internet Explorer
     - id: ie4.01-mac
@@ -79,27 +88,31 @@ browsers:
       version: '4.01'
       icon: 'mac-ie4.png'
       path: ie4mac
+      about: 'https://en.wikipedia.org/wiki/Internet_Explorer_for_Mac#Internet_Explorer_4.0_for_Macintosh'
 
     - id: ie5.1-mac
       name: 'Internet Explorer'
       os: 'Macintosh'
       version: '5.1.7'
       icon: 'mac-ie5.1.png'
       path: ie5mac
+      about: 'https://en.wikipedia.org/wiki/Internet_Explorer_for_Mac#Internet_Explorer_5_Macintosh_Edition'
 
     - id: ie4
       name: 'Internet Explorer'
       os: 'Windows'
       version: '4.01'
       icon: 'win32-ie4.png'
       path: ie4
+      about: 'https://en.wikipedia.org/wiki/Internet_Explorer_4'
 
     - id: ie5.5
       name: 'Internet Explorer'
       os: 'Windows'
       version: '5.5'
       icon: 'win32-ie5.5.png'
       path: ie5.5
+      about: 'https://en.wikipedia.org/wiki/Internet_Explorer_5'
 
     # Safari
     - id: safari5
@@ -108,3 +121,4 @@ browsers:
       version: '5.0'
       icon: 'safari.png'
       path: safari
+      about: 'https://en.wikipedia.org/wiki/Safari_%28web_browser%29#Safari_5'

app/main.py

@@ -232,6 +232,7 @@ def route_load_url(path='', url='', ts=''):
     browser_info = dict(name=browser['name'],
                         os=browser['os'],
                         version=browser['version'],
+                        about=browser['about'],
                         icon=browser['icon'])
 
     return {'coll': path,

app/static/main.css

@@ -153,6 +153,12 @@ span.purple{color:purple}
         text-align: center;
     }
 
+    .about-browser {
+        font-size: 12px;
+        text-align: left;
+        margin-left: 4px;
+    }
+
 
     /* Button that opens either browser or datetime selector */
     .dropdown {