use local projectdb in bench test, try to fix python 3.3 test hang issue

pyspider/libs/bench.py

@@ -4,6 +4,8 @@
 # Author: Binux<[email protected]>
 #         http://binux.me
 # Created on 2014-12-08 22:23:10
+# rate: 10000000000
+# burst: 10000000000
 
 import time
 import logging
@@ -248,17 +250,16 @@ def on_result(self, task, result):
         super(BenchResultWorker, self).on_result(task, result)
 
 
-bench_script = '''
-from pyspider.libs.base_handler import *
+from pyspider.libs.base_handler import BaseHandler
+
 
 class Handler(BaseHandler):
-    def on_start(self):
+    def on_start(self, response):
         self.crawl('http://127.0.0.1:5000/bench',
-                   params={'total': %(total)d, 'show': %(show)d},
+                   params={'total': response.save.get('total', 10000), 'show': response.save.get('show', 20)},
                    callback=self.index_page)
 
     def index_page(self, response):
         for each in response.doc('a[href^="http://"]').items():
             self.crawl(each.attr.href, callback=self.index_page)
         return response.url
-'''

pyspider/processor/project_module.py

@@ -266,3 +266,6 @@ def create_module(self, spec):
 
         def exec_module(self, module):
             return module
+
+        def module_repr(self, module):
+            return '<Module projects.%s>' % self.name

pyspider/run.py

@@ -117,9 +117,9 @@ def cli(ctx, **kwargs):
                 os.mkdir(kwargs['data_path'])
             if db in ('taskdb', 'resultdb'):
                 kwargs[db] = utils.Get(lambda db=db: connect_database('sqlite+%s://' % (db)))
-            else:
-                kwargs[db] = utils.Get(lambda db=db: connect_database('sqlite+%s:///%s/%s.db' % (
-                    db, kwargs['data_path'], db[:-2])))
+            elif db in ('projectdb', ):
+                kwargs[db] = utils.Get(lambda db=db: connect_database('local+%s://%s' % (
+                    db, os.path.join(os.path.dirname(__file__), 'libs/bench.py'))))
         else:
             if not os.path.exists(kwargs['data_path']):
                 os.mkdir(kwargs['data_path'])
@@ -556,22 +556,13 @@ def bench(ctx, fetcher_num, processor_num, result_worker_num, run_in, total, sho
     if not all_test and not all_bench:
         return
 
-    project_name = '__bench_test__'
+    project_name = 'bench'
 
     def clear_project():
         g.taskdb.drop(project_name)
-        g.projectdb.drop(project_name)
         g.resultdb.drop(project_name)
 
     clear_project()
-    g.projectdb.insert(project_name, {
-        'name': project_name,
-        'status': 'RUNNING',
-        'script': bench.bench_script % {'total': total, 'show': show},
-        'rate': total,
-        'burst': total,
-        'updatetime': time.time()
-    })
 
     # disable log
     logging.getLogger().setLevel(logging.ERROR)
@@ -632,6 +623,9 @@ def clear_project():
             "project": project_name,
             "taskid": "on_start",
             "url": "data:,on_start",
+            "fetch": {
+                "save": {"total": total, "show": show}
+            },
             "process": {
                 "callback": "on_start",
             },