rewrite django pages

search/admin.py

@@ -1,9 +1,9 @@
 from django.contrib import admin
-from search.models import Hash, FileList, StatusReport
+from search.models import Hash, FileList, StatusReport, RecKeywords
 
 
 # Register your models here.
 admin.site.register(Hash)
 admin.site.register(FileList)
 admin.site.register(StatusReport)
-
+admin.site.register(RecKeywords)

search/migrations/0005_auto_20150721_0628.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('search', '0004_auto_20150511_0339'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RecKeywords',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('keyword', models.CharField(max_length=20, verbose_name=b'\xe6\x8e\xa8\xe8\x8d\x90\xe5\x85\xb3\xe9\x94\xae\xe8\xaf\x8d')),
+                ('order', models.PositiveIntegerField(verbose_name=b'\xe6\x8e\x92\xe5\xba\x8f')),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='filelist',
+            name='file_list',
+            field=models.TextField(verbose_name=b'JSON\xe6\xa0\xbc\xe5\xbc\x8f\xe7\x9a\x84\xe6\x96\x87\xe4\xbb\xb6\xe5\x88\x97\xe8\xa1\xa8'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='category',
+            field=models.CharField(max_length=20, verbose_name=b'\xe7\xb1\xbb\xe5\x88\xab'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='classified',
+            field=models.BooleanField(default=False, verbose_name=b'\xe6\x98\xaf\xe5\x90\xa6\xe5\x88\x86\xe7\xb1\xbb'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='create_time',
+            field=models.DateTimeField(verbose_name=b'\xe5\x85\xa5\xe5\xba\x93\xe6\x97\xb6\xe9\x97\xb4'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='data_hash',
+            field=models.CharField(max_length=32, verbose_name=b'\xe5\x86\x85\xe5\xae\xb9\xe7\xad\xbe\xe5\x90\x8d'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='extension',
+            field=models.CharField(max_length=20, verbose_name=b'\xe6\x89\xa9\xe5\xb1\x95\xe5\x90\x8d'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='last_seen',
+            field=models.DateTimeField(verbose_name=b'\xe4\xb8\x8a\xe6\xac\xa1\xe8\xaf\xb7\xe6\xb1\x82\xe6\x97\xb6\xe9\x97\xb4'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='length',
+            field=models.BigIntegerField(verbose_name=b'\xe6\x96\x87\xe4\xbb\xb6\xe5\xa4\xa7\xe5\xb0\x8f'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='name',
+            field=models.CharField(max_length=255, verbose_name=b'\xe8\xb5\x84\xe6\xba\x90\xe5\x90\x8d\xe7\xa7\xb0'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='requests',
+            field=models.PositiveIntegerField(verbose_name=b'\xe8\xaf\xb7\xe6\xb1\x82\xe6\xac\xa1\xe6\x95\xb0'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='source_ip',
+            field=models.CharField(max_length=20, null=True, verbose_name=b'\xe6\x9d\xa5\xe6\xba\x90IP'),
+        ),
+        migrations.AlterField(
+            model_name='hash',
+            name='tagged',
+            field=models.BooleanField(default=False, db_index=True, verbose_name=b'\xe6\x98\xaf\xe5\x90\xa6\xe7\xb4\xa2\xe5\xbc\x95'),
+        ),
+        migrations.AlterField(
+            model_name='statusreport',
+            name='date',
+            field=models.DateField(unique=True, verbose_name=b'\xe6\x97\xa5\xe6\x9c\x9f'),
+        ),
+        migrations.AlterField(
+            model_name='statusreport',
+            name='new_hashes',
+            field=models.IntegerField(verbose_name=b'\xe6\x96\xb0\xe5\xa2\x9e\xe8\xb5\x84\xe6\xba\x90'),
+        ),
+    ]

search/models.py

@@ -1,20 +1,58 @@
+#coding: utf8
+import json
+import binascii
+
 from django.db import models
+from sphinxit.core.helpers import BaseSearchConfig
+from sphinxit.core.processor import Search
+from sphinxit.core.nodes import Count
+
+
+class SphinxitConfig(BaseSearchConfig):
+    WITH_STATUS = False
 
 
-# Create your models here.
+class HashManager(models.Manager):
+    def search(self, keyword, start=0, count=10, category=None, sort=None):
+        search_query = Search(indexes=['rt_main'], config=SphinxitConfig)
+        q = search_query.match(keyword)
+        if category: q = q.filter(category__eq=binascii.crc32(category)&0xFFFFFFFFL)
+        if sort == 'create_time': q = q.order_by('create_time', 'desc')
+        if sort == 'length': q = q.order_by('length', 'desc')
+        q = q.limit(start, count)
+        q2 = search_query.match(keyword).select('category', Count()).group_by('category').named('cats')
+        res = q.ask(subqueries=[q2])
+        return res
+
+    def list_with_files(self, ids):
+        res = []
+        if len(ids[0]) == 40:
+            items = Hash.objects.filter(info_hash__in=ids).values()
+        else:
+            items = Hash.objects.filter(id__in=ids).values()
+        res = list(items)
+        items = FileList.objects.filter(info_hash__in=[x['info_hash'] for x in res]).values()
+        for x in res:
+            for y in items:
+                if x['info_hash'] == y['info_hash']:
+                    x['files'] = json.loads(y['file_list'])
+        return res
+
 class Hash(models.Model):
+    objects = HashManager()
+
     info_hash = models.CharField(max_length=40, unique=True)
-    category = models.CharField(max_length=20)
-    data_hash = models.CharField(max_length=32)
-    name = models.CharField(max_length=255)
-    extension = models.CharField(max_length=20)
-    classified = models.BooleanField(default=False)
-    source_ip = models.CharField(max_length=20, null=True)
-    tagged = models.BooleanField(default=False, db_index=True)
-    length = models.BigIntegerField()
-    create_time = models.DateTimeField()
-    last_seen = models.DateTimeField()
-    requests = models.PositiveIntegerField()
+    category = models.CharField('类别', max_length=20)
+    data_hash = models.CharField('内容签名', max_length=32)
+    name = models.CharField('资源名称', max_length=255)
+    extension = models.CharField('扩展名', max_length=20)
+    classified = models.BooleanField('是否分类', default=False)
+    source_ip = models.CharField('来源IP', max_length=20, null=True)
+    tagged = models.BooleanField('是否索引', default=False, db_index=True)
+    length = models.BigIntegerField('文件大小')
+    create_time = models.DateTimeField('入库时间')
+    last_seen = models.DateTimeField('上次请求时间')
+    requests = models.PositiveIntegerField('请求次数')
     comment = models.CharField(max_length=255, null=True)
     creator = models.CharField(max_length=20, null=True)
 
@@ -24,18 +62,27 @@ def __unicode__(self):
 
 class FileList(models.Model):
     info_hash = models.CharField(max_length=40, primary_key=True)
-    file_list = models.TextField()
+    file_list = models.TextField('JSON格式的文件列表')
 
     def __unicode__(self):
         return self.info_hash
 
 
 class StatusReport(models.Model):
-    date = models.DateField(unique=True)
-    new_hashes = models.IntegerField()
+    date = models.DateField('日期', unique=True)
+    new_hashes = models.IntegerField('新增资源')
     total_requests = models.IntegerField()
     valid_requests = models.IntegerField()
 
     def __unicode__(self):
         return u'%s - %s' % (self.date, self.new_hashes)
 
+
+class RecKeywords(models.Model):
+    keyword = models.CharField('推荐关键词', max_length=20)
+    order = models.PositiveIntegerField('排序', default=0)
+
+    def __unicode__(self):
+        return u'%s %s' % (self.keyword, self.order)
+
+

search/timermiddleware.py

@@ -0,0 +1,17 @@
+from time import time
+
+class TimerMiddleware:
+    def process_request(self, request):
+        request._tm_start_time = time()
+
+    def process_response(self, request, response):
+        if not hasattr(request, "_tm_start_time"):
+            return
+
+        total = (time() - request._tm_start_time) * 1000
+
+        if total > 200:
+            print 'Long time request %10sms %s' % (total, request.path)
+        response['X-Request-Time'] = '%fsms' % total
+        return response
+

search/urls.py

@@ -0,0 +1,12 @@
+from django.conf.urls import include, url
+
+from .views import json_status, json_info, json_search
+
+
+urlpatterns = [
+    url(r'^json_search', json_search),
+    url(r'^json_info', json_info),
+    url(r'^json_status', json_status),
+]
+
+

search/views.py

@@ -1,77 +1,39 @@
 #coding: utf8
+import traceback
+
 from django.shortcuts import render
-from ssbc import settings
+from django.views.decorators.cache import cache_page
 from django.http import JsonResponse, HttpResponse
-from search.models import Hash, FileList, StatusReport
-from sphinxit.core.helpers import BaseSearchConfig
-from sphinxit.core.processor import Search
-from sphinxit.core.nodes import Count
-import binascii
-import json
-import memcache
 
-
-class SphinxitConfig(BaseSearchConfig):
-    WITH_STATUS = False
-mc = memcache.Client(['127.0.0.1:11211'],debug=0)
+from ssbc import settings
+from search.models import Hash, FileList, StatusReport
 
 
+@cache_page(600)
 def json_search(request):
-    search_query = Search(indexes=['rt_main'], config=SphinxitConfig)
     keyword = request.GET['keyword']
     start = request.GET.get('start', 0)
     count = request.GET.get('count', 10)
     sort = request.GET.get('sort', '')
     category = request.GET.get('category', '')
     if request.GET.get('base64') == '1':
         keyword = keyword.decode('base64').decode('utf8')
-
-    mckey = str(binascii.crc32((u'%s%s%s%s%s' % (keyword,start,count,sort,category)).encode('utf8')) & 0xFFFFFFFFL)
-    cache = mc.get(mckey)
-    if cache:
-        print 'bingo', keyword.encode('utf8'), mckey
-        return HttpResponse(cache)
-
-    q = search_query.match(keyword)
-    if category: q = q.filter(category__eq=binascii.crc32(category)&0xFFFFFFFFL)
-    if sort == 'create_time': q = q.order_by('create_time', 'desc')
-    if sort == 'length': q = q.order_by('length', 'desc')
-    q = q.limit(start, count)
-    q2 = search_query.match(keyword).select('category', Count()).group_by('category').named('cats')
-    res = q.ask(subqueries=[q2])
-
-    jsp = JsonResponse(res)
-    mc.set(mckey, jsp.content)
-    return jsp
+    res = Hash.objects.search(keyword, start, count, category, sort)
+    return JsonResponse(res)
 
 
+@cache_page(600)
 def json_info(request):
     hashes = request.GET['hashes']
-    mckey = str(binascii.crc32(str(hashes)) & 0xFFFFFFFFL)
-    cache = mc.get(mckey)
-    if cache:
-        return HttpResponse(cache)
-
-    res = {}
-    for h in hashes.split('-'):
-        if len(h) == 40:
-            item = Hash.objects.filter(info_hash=h).values()
-        else:
-            item = Hash.objects.filter(id=h).values()
-        try:
-            res[h] = item[0]
-        except:
-            return HttpResponse('{}')
-        try:
-            filelist = FileList.objects.get(info_hash=res[h]['info_hash'])
-            res[h]['files'] = json.loads(filelist.file_list)
-        except:
-            pass
-    jsp = JsonResponse(res)
-    mc.set(mckey, jsp.content)
-    return jsp
+    try:
+        res = Hash.objects.list_with_files(hashes.split('-'))
+        j = {'result': res, 'ret': 0}
+    except:
+        j = {'ret': 1, 'error': traceback.format_exc()}
+    return JsonResponse(j)
 
 
+@cache_page(600)
 def json_status(request):
     d = {}
     d['hash_total'] = Hash.objects.count()

ssbc/settings.py

@@ -50,6 +50,7 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.security.SecurityMiddleware',
+    'search.timermiddleware.TimerMiddleware',
 )
 
 ROOT_URLCONF = 'ssbc.urls'
@@ -110,3 +111,10 @@
 STATIC_URL = '/static/'
 STATIC_ROOT = os.path.join(BASE_DIR, 'www/static')
 
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'LOCATION': '127.0.0.1:11211',
+    }
+}
+

ssbc/urls.py

@@ -16,19 +16,16 @@
 from django.conf.urls import include, url
 from django.contrib import admin
 
-import search.views
 import web.views
 
 urlpatterns = [
     url(r'^admin/', include(admin.site.urls)),
-    url(r'^api/json_search', search.views.json_search),
-    url(r'^api/json_info', search.views.json_info),
-    url(r'^api/json_status', search.views.json_status),
+    url(r'^api/', include('search.urls')),
     url(r'^$', web.views.index, name='index'),
     url(r'^h/(\d{1,10})$', web.views.hash, name='hash'),
     url(r'^hash/(.{40})$', web.views.hash),
     url(r'^info/(.{40})$', web.views.hash_old),
-    url(r'^search/(.+?)/(\d*)$', web.views.search_list, name='list'),
+    url(r'^search/(.+?)/(\d*)$', web.views.search, name='list'),
     url(r'^search/(.+?)$', web.views.search),
     url(r'^list/(.+?)/(\d*)$', web.views.search_old),
     url(r'^howto/$', web.views.howto, name='howto'),

web/templates/index.html

@@ -23,8 +23,8 @@
         </div>
         <div class="good">
             <p>
-            {% for kw in reclist%}
-                <a href="/search/{{kw|urlencode}}/">{{kw}}</a> &nbsp;&nbsp;&nbsp;&nbsp;
+            {% for r in reclist%}
+                <a href="/search/{{r.keyword|urlencode}}/">{{r.keyword}}</a> &nbsp;&nbsp;&nbsp;&nbsp;
             {% endfor %}
             </p>
             <p class="tips2">

web/templatetags/filters.py

@@ -18,7 +18,10 @@ def smartcoffee(value, autoescape=True):
 
 @register.filter()
 def format_time(t):
-    d = datetime.datetime.strptime(t.split('.')[0], '%Y-%m-%dT%H:%M:%S')
+    if type(t) is datetime.datetime:
+        d = t
+    else:
+        d = datetime.datetime.strptime(t.split('.')[0], '%Y-%m-%dT%H:%M:%S')
     now = datetime.datetime.utcnow()
     dt = now - d
     if dt.days > 365: