Refactoring mongoengine models

app.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+from flask import Flask
+
+import config
+from ext import db, mako
+
+
+def create_app():
+    app = Flask(__name__, template_folder='templates',
+                static_folder='static')
+    app.config.from_object(config)
+    mako.init_app(app)
+    db.init_app(app)
+    return app
+
+
+if __name__ == '__main__':
+    app = create_app()
+    app.run(host='0.0.0.0', port=8000, debug=app.debug)

assets/.eslintrc

@@ -0,0 +1,30 @@
+{
+  "parser": "babel-eslint",
+  "extends": "eslint:recommended",
+  "ecmaFeatures": {
+    "jsx": true,
+    "modules": true
+  },
+  "rules": {
+    "no-console": 1,
+    "no-unused-vars": [1, {"vars": "all", "args": "after-used", "varsIgnorePattern": "^_"}],
+    "curly": 2,
+    "eqeqeq": [2, "smart"],
+    "react/jsx-no-undef": 1,
+    "react/jsx-pascal-case": 1,
+    "react/jsx-uses-vars": 1,
+    "react/jsx-uses-react": 1
+  },
+  "env": {
+    "browser": true,
+    "commonjs": true,
+    "es6": true
+  },
+  "globals": {
+    "process": true,
+    "__DEV__": true
+  },
+  "plugins": [
+    "react"
+  ]
+}

assets/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "assets",
+  "version": "1.0.0",
+  "description": "Musicbox",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dongweiming/musicbox.git"
+  },
+  "keywords": [
+    "react"
+  ],
+  "author": "Dongweiming",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/dongweiming/musicbox/issues"
+  },
+  "homepage": "https://github.com/dongweiming/musicbox#readme",
+  "dependencies": {
+    "material-design-icons": "^3.0.1",
+    "material-ui": "^0.15.4",
+    "mobx": "^2.5.1",
+    "mobx-react": "^3.5.5",
+    "react": "^15.3.1",
+    "react-dom": "^15.3.1",
+    "react-router": "^2.8.0",
+    "react-tap-event-plugin": "^1.0.0"
+  },
+  "devDependencies": {
+    "babel-core": "^6.14.0",
+    "babel-loader": "^6.2.5",
+    "babel-plugin-transform-class-properties": "^6.11.5",
+    "babel-plugin-transform-decorators-legacy": "^1.3.4",
+    "babel-preset-es2015": "^6.14.0",
+    "babel-preset-react": "^6.11.1",
+    "body-parser": "^1.15.2",
+    "colors": "^1.1.2",
+    "eslint": "^3.5.0",
+    "eslint-loader": "^1.5.0",
+    "eslint-plugin-react": "^6.2.0",
+    "express": "^4.14.0",
+    "http-proxy-middleware": "^0.17.1",
+    "mobx-react-devtools": "^4.2.6",
+    "react-hot-loader": "^3.0.0-beta.3",
+    "serve-index": "^1.8.0",
+    "webpack": "^1.13.2",
+    "webpack-dev-middleware": "^1.7.0",
+    "webpack-dev-server": "^1.15.1",
+    "webpack-hot-middleware": "^2.12.2"
+  },
+  "babel": {
+    "presets": [
+      "es2015",
+      "react",
+      "stage-2"
+    ],
+    "plugins": [
+      "transform-decorators-legacy",
+      "transform-class-properties"
+    ]
+  }
+}

assets/server.js

@@ -0,0 +1,69 @@
+var colors = require('colors');
+var webpack = require('webpack');
+var express = require('express');
+var WebpackDevServer = require('webpack-dev-server');
+var bodyParser = require('body-parser');
+var httpProxyMiddleware = require("http-proxy-middleware");
+var serveIndex = require("serve-index");
+var config = require('./webpack.config.js');
+
+config.debug = true;
+config.devtool = 'eval';
+Object.keys(config.entry).forEach(key => {
+    var item = Array.prototype.slice.call(config.entry[key]);
+    item.unshift(
+        'webpack-hot-middleware/client?path=/__webpack_hmr',
+        'webpack/hot/only-dev-server'
+    );
+});
+
+config.plugins = config.plugins.concat(
+  new webpack.HotModuleReplacementPlugin(),
+  new webpack.NoErrorsPlugin()
+);
+
+var compiler = webpack(config);
+var app = new WebpackDevServer(compiler, {
+    hot: true,
+    filename: config.output.filename,
+    publicPath: config.output.publicPath,
+    stats: {
+        assets: false,
+        colors: true,
+        chunks: false,
+        children: false
+    }
+}).app;
+
+app.use(require('webpack-hot-middleware')(compiler, {
+    log: console.log,
+    path: '/__webpack_hmr',
+    heartbeat: 10 * 1000
+}));
+
+app.use(httpProxyMiddleware('**', {
+  target: 'http://localhost:8000'
+}));
+
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.get('/:appName', function(req, res) {
+    var appName = req.params.appName;
+    res.render('index', config.appConfigs[appName]);
+});
+
+var contentBase = '.';
+app.get('*', express.static(contentBase), serveIndex(contentBase));
+
+app.listen(3000, '0.0.0.0', function(err) {
+    if (err) {
+        console.log(err);
+        return;
+    }
+    console.log('[webpack-dev-server]'.magenta);
+    console.log('* You can visit:'.green);
+    Object.keys(config.entry).forEach(appName => {
+        console.log('   > ' + ('http://0.0.0.0:3000/' + appName).underline.cyan);
+    });
+});

assets/webpack.config.js

@@ -0,0 +1,46 @@
+var fs = require('fs');
+var path = require('path');
+var webpack = require('webpack');
+
+var output = path.join(__dirname, '..');
+var nodeModulePath = path.join(__dirname, 'node_modules');
+var publicPath = '/static/js/dist/';
+
+var plugins = [
+    new webpack.DefinePlugin({
+        __DEV__: process.env.NODE_ENV !== 'production',
+        'process.env.NODE_ENV': JSON.stringify(process.env.NODE_ENV)
+    })
+];
+
+module.exports = {
+    devtool: "source-map",
+    entry: {
+		    index: "./src/index"
+	  },
+    output: {
+        path: path.join(output, publicPath),
+        filename: '[name].bundle.js',
+        publicPath: publicPath
+    },
+    resolve: {
+        extensions: ['', '.js', '.es6']
+    },
+    module: {
+        preLoaders: [
+            {
+                test: /\.es6?$/,
+                exclude: /node_modules/,
+                loader: 'eslint'
+            }
+        ],
+        loaders: [
+            {
+                test: /(\.es6|\.js)$/,
+                exclude: /node_modules/,
+                loader: 'babel'
+            }
+        ]
+    },
+    plugins: plugins
+};

config.py

@@ -1,12 +1,14 @@
 import os
 
-DB_HOST = '127.0.01'
-DB_PORT = 27017
-DATABASE_NAME = 'music'
+MONGODB_DB = 'music'
+MONGODB_HOST = 'localhost'
+MONGODB_PORT = 27017
 PROXIES = []
 
 HERE = os.path.abspath(os.path.dirname(__file__))
-DB = os.path.join(HERE, 'data/fake_useragent.json')
+DATA_DB = os.path.join(HERE, 'data/fake_useragent.json')
+
+DEBUG = False
 
 try:
     from local_settings import *  # noqa

ext.py

@@ -0,0 +1,7 @@
+# coding=utf-8
+from flask_mongoengine import MongoEngine
+from flask_mako import MakoTemplates, render_template  # noqa
+
+db = MongoEngine()
+mako = MakoTemplates()
+

models.py

@@ -1,22 +1,17 @@
 from datetime import datetime
 
-from mongoengine import (Document, StringField, DateTimeField, ReferenceField,
-                         IntField, ListField, DoesNotExist, connect)
-from config import DB_HOST, DB_PORT, DATABASE_NAME
+from mongoengine.queryset import DoesNotExist
+from ext import db
 
 ARTIST_URL = 'http://music.163.com/#/artist?id={}'
 SONG_URL = 'http://music.163.com/#/song?id={}'
 USER_URL = 'http://music.163.com/#/user/home?id={}'
 
 
-def lazy_connect():
-    connect(DATABASE_NAME, host=DB_HOST, port=DB_PORT)
-
-
-class BaseModel(Document):
-    id = IntField(primary_key=True)
-    create_at = DateTimeField(default=datetime.now)
-    update_at = DateTimeField()
+class BaseModel(db.Document):
+    id = db.IntField(primary_key=True)
+    create_at = db.DateTimeField(default=datetime.now)
+    update_at = db.DateTimeField()
 
     meta = {'allow_inheritance': True,
             'abstract': True
@@ -34,20 +29,20 @@ def get_or_create(cls, **kwargs):
 
 
 class Artist(BaseModel):
-    name = StringField()
-    picture = StringField()
-    songs = ListField(ReferenceField('Song'))
+    name = db.StringField()
+    picture = db.StringField()
+    songs = db.ListField(db.ReferenceField('Song'))
 
     @property
     def url(self):
         return ARTIST_URL.format(self.id)
 
 
 class Song(BaseModel):
-    name = StringField()
-    comment_count = IntField()
-    comments = ListField(ReferenceField('Comment'))
-    artist = ReferenceField('Artist')
+    name = db.StringField()
+    comment_count = db.IntField()
+    comments = db.ListField(db.ReferenceField('Comment'))
+    artist = db.ReferenceField('Artist')
 
     @property
     def url(self):
@@ -59,10 +54,10 @@ def artist_url(self):
 
 
 class Comment(BaseModel):
-    content = StringField()
-    like_count = IntField()
-    user = ReferenceField('User')
-    song = ReferenceField('Song')
+    content = db.StringField()
+    like_count = db.IntField()
+    user = db.ReferenceField('User')
+    song = db.ReferenceField('Song')
 
     @property
     def user_url(self):
@@ -78,8 +73,8 @@ def user_url(self):
 
 
 class User(BaseModel):
-    name = StringField()
-    picture = StringField()
+    name = db.StringField()
+    picture = db.StringField()
 
     @property
     def url(self):
@@ -88,7 +83,7 @@ def url(self):
 
 class Process(BaseModel):
     STATUS = PENDING, SUCCEEDED, FAILED = range(3)
-    status = IntField(choices=STATUS, default=PENDING)
+    status = db.IntField(choices=STATUS, default=PENDING)
 
     @property
     def is_success(self):

spider/encrypt.py

@@ -5,9 +5,13 @@
 import json
 
 if platform.system() == 'Darwin':
-    import crypto
-    import sys
-    sys.modules['Crypto'] = crypto
+    try:
+        import crypto
+        import sys
+        sys.modules['Crypto'] = crypto
+    except ImportError:
+        pass
+
 
 from Crypto.Cipher import AES
 

spider/parser.py

@@ -2,7 +2,8 @@
 
 from mongoengine.connection import disconnect
 
-from models import Artist, Song, Comment, User, Process, lazy_connect
+from models import Artist, Song, Comment, User, Process
+from app import create_app
 
 from spider.utils import get_user_agent, get_tree, post
 
@@ -26,8 +27,7 @@ def unprocess_artist_list():
 
 
 def parser_artist(artist_id):
-    disconnect()
-    lazy_connect()
+    create_app()
     process = Process.get_or_create(id=artist_id)
     if process.is_success:
         return

views.py

@@ -0,0 +1,4 @@
+# coding=utf-8
+from flask import Flask
+
+