-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Xiaoxia
committed
May 14, 2015
1 parent
0127a5f
commit 7a14ecb
Showing
4 changed files
with
30 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,13 @@ | ||
# ssbc | ||
手撕包菜网站 | ||
|
||
## 创建MySQL数据库 | ||
create database ssbc default charset utf8; | ||
## 网站说明 | ||
这是 www.shousibaocai.com 的网站源代码。 | ||
开源的目的是为了促进技术交流和相互学习,把DHT与搜索引擎技术应用到更广泛的领域去。 | ||
|
||
本站于2015年5月使用django改写。 | ||
与爬虫相关的代码都在目录workers下。 | ||
|
||
相关文章请查看作者博客: | ||
http://xiaoxia.org/2015/05/14/shousibaocai-opensource/ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,12 @@ | ||
#!/usr/bin/env python | ||
#coding: utf8 | ||
""" | ||
磁力搜索爬虫代码 | ||
爬虫获取的hash地址会放进队列,由负责下载资源信息的meta_worker处理和入库。 | ||
[email protected] | ||
2013.5 created | ||
2015.5 updated | ||
""" | ||
|
||
import bencode | ||
import sys | ||
|
@@ -19,7 +26,7 @@ | |
from SimpleXMLRPCServer import SimpleXMLRPCServer | ||
|
||
DHT_PORT = 6881 | ||
DHT_IP = '98.126.37.226' | ||
DHT_IP = socket.gethostbyname(socket.gethostname()) | ||
|
||
req_queue = Queue.Queue(1000) | ||
|
||
|
@@ -38,7 +45,7 @@ def __init__(self): | |
break | ||
except: | ||
DHT_PORT += 1 | ||
print 'Bind socket at port', DHT_PORT | ||
print 'Bind socket at %s:%s' % (DHT_IP, DHT_PORT) | ||
self.node_id = self.get_random_id() | ||
self.version = 'XTxx' | ||
self.new_nodes = Queue.Queue() | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,10 @@ | ||
#!/usr/bin/env python | ||
#coding: utf8 | ||
""" | ||
从MySQL数据库中读取未索引的资源,更新到Sphinx的实时索引中。 | ||
[email protected] | ||
2015.5 created | ||
""" | ||
|
||
import time | ||
import MySQLdb as mdb | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,13 @@ | ||
#!/usr/bin/env python2.6 | ||
#coding: utf8 | ||
""" | ||
磁力搜索meta信息入库程序 | ||
从队列里读取hash,如果数据库里不存在,就使用libtorrent获取meta信息,保存到数据库中。 | ||
[email protected] | ||
2013.5 created | ||
2015.5 updated | ||
""" | ||
|
||
|
||
import cPickle | ||
import hashlib | ||
|