Merge pull request crawlab-team#548 from crawlab-team/develop

Develop

CHANGELOG-zh.md

@@ -1,3 +1,14 @@
+# 0.4.6 (未知)
+### 功能 / 优化
+- **Node.js SDK **. 用户可以将 SDK 应用到他们的 Node.js 爬虫中.
+- **日志管理优化**. 日志搜索，错误高亮，自动滚动.
+- **任务执行流程优化**. 允许用户在触发任务后跳转到该任务详情页.
+- **任务展示优化**. 在爬虫详情页的最近任务表格中加入了“参数”列. [#295](https://github.com/crawlab-team/crawlab/issues/295)
+
+### Bug 修复
+- **定时任务配置失去焦点**. [#519](https://github.com/crawlab-team/crawlab/issues/519)
+- **无法用 CLI 工具上传爬虫**. [#524](https://github.com/crawlab-team/crawlab/issues/524)
+
 # 0.4.5 (2020-02-03)
 ### 功能 / 优化
 - **交互式教程**. 引导用户了解 Crawlab 的主要功能.

CHANGELOG.md

@@ -1,3 +1,14 @@
+# 0.4.6 (unkown)
+### Features / Enhancement
+- **SDK for Node.js**. Users can apply SDK in their Node.js spiders.
+- **Log Management Optimization**. Log search, error highlight, auto-scrolling.
+- **Task Execution Process Optimization**. Allow users to be redirected to task detail page after triggering a task.
+- **Task Display Optimization**. Added "Param" in the Latest Tasks table in the spider detail page. [#295](https://github.com/crawlab-team/crawlab/issues/295)
+
+### Bug Fixes
+- **Lost Focus in Schedule Configuration**. [#519](https://github.com/crawlab-team/crawlab/issues/519)
+- **Unable to Upload Spider using CLI**. [#524](https://github.com/crawlab-team/crawlab/issues/524)
+
 # 0.4.5 (2020-02-03)
 ### Features / Enhancement
 - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab.

docker-compose.yml

@@ -35,6 +35,8 @@ services:
     depends_on:
       - mongo
       - redis
+    # volumes:
+    #   - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
   worker:
     image: tikazyq/crawlab:latest
     container_name: worker
@@ -45,6 +47,8 @@ services:
     depends_on:
       - mongo
       - redis
+    # volumes:
+    #   - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
   mongo:
     image: mongo:latest
     restart: always

frontend/src/components/ScrollView/LogItem.vue

@@ -1,9 +1,13 @@
 <template>
-  <div class="log-item">
+  <div class="log-item" :style="style">
     <div class="line-no">{{index}}</div>
     <div class="line-content">
-      <span v-if="isAnsi" v-html="dataHtml"></span>
-      <span v-else="" v-html="dataHtml"></span>
+      <span v-if="isLogEnd" style="color: #E6A23C" class="loading-text">
+        {{$t('Updating log...')}}
+        <i class="el-icon-loading"></i>
+      </span>
+      <span v-else-if="isAnsi" v-html="dataHtml"></span>
+      <span v-else v-html="dataHtml"></span>
     </div>
   </div>
 </template>
@@ -29,10 +33,31 @@ export default {
       default: ''
     }
   },
+  data () {
+    const token = ' :,.'
+    return {
+      errorRegex: new RegExp(`(?:[${token}]|^)((?:error|exception|traceback)s?)(?:[${token}]|$)`, 'gi')
+      // errorRegex: new RegExp('(error|exception|traceback)', 'gi')
+    }
+  },
   computed: {
     dataHtml () {
-      if (!this.searchString) return this.data
-      return this.data.replace(new RegExp(`(${this.searchString})`, 'gi'), '<mark>$1</mark>')
+      let html = this.data.replace(this.errorRegex, ' <span style="font-weight: bolder; text-decoration: underline">$1</span> ')
+      if (!this.searchString) return html
+      html = html.replace(new RegExp(`(${this.searchString})`, 'gi'), '<mark>$1</mark>')
+      return html
+    },
+    style () {
+      let color = ''
+      if (this.data.match(this.errorRegex)) {
+        color = '#F56C6C'
+      }
+      return {
+        color
+      }
+    },
+    isLogEnd () {
+      return this.data === '###LOG_END###'
     }
   }
 }
@@ -64,4 +89,18 @@ export default {
     word-break: break-word;
     flex-basis: calc(100% - 50px);
   }
+
+  .loading-text {
+    animation: blink;
+  }
+
+  @keyframes blink {
+    0% {
+      opacity: 0;
+    }
+
+    100% {
+      opacity: 100%;
+    }
+  }
 </style>

frontend/src/components/ScrollView/LogView.vue

@@ -1,30 +1,50 @@
 <template>
   <div class="log-view-wrapper">
     <div class="filter-wrapper">
+      <el-button
+        size="small"
+        type="primary"
+        icon="el-icon-download"
+        style="margin-right: 10px"
+        :disabled="isToBottom"
+        @click="onAutoScroll"
+      >
+        {{$t('Auto-Scroll')}}
+      </el-button>
       <el-input
         v-model="searchString"
+        size="small"
         suffix-icon="el-icon-search"
         :placeholder="$t('Search Log')"
         style="width: 240px"
       />
     </div>
-    <virtual-list
-      class="log-view"
-      :size="6"
-      :remain="100"
-      :item="item"
-      :itemcount="filteredLogData.length"
-      :itemprops="getItemProps"
-    />
+    <div class="log-view-wrapper" ref="log-view-wrapper">
+      <virtual-list
+        class="log-view"
+        ref="log-view"
+        :size="6"
+        :remain="100"
+        :item="item"
+        :itemcount="filteredLogData.length"
+        :itemprops="getItemProps"
+        :tobottom="onToBottom"
+        :onscroll="onScroll"
+      />
+    </div>
   </div>
 </template>
 
 <script>
-import LogItem from './LogItem'
+import {
+  mapState
+} from 'vuex'
 import VirtualList from 'vue-virtual-scroll-list'
 import Convert from 'ansi-to-html'
 import hasAnsi from 'has-ansi'
 
+import LogItem from './LogItem'
+
 const convert = new Convert()
 export default {
   name: 'LogView',
@@ -40,18 +60,35 @@ export default {
   data () {
     return {
       item: LogItem,
-      searchString: ''
+      searchString: '',
+      isToBottom: false,
+      isScrolling: false,
+      isScrolling2nd: false
     }
   },
   computed: {
+    ...mapState('task', [
+      'taskForm'
+    ]),
     logData () {
-      return this.data.split('\n')
+      const data = this.data.split('\n')
         .map((d, i) => {
           return {
             index: i + 1,
             data: d
           }
         })
+      if (this.taskForm && this.taskForm.status === 'running') {
+        data.push({
+          index: data.length + 1,
+          data: '###LOG_END###'
+        })
+        data.push({
+          index: data.length + 2,
+          data: ''
+        })
+      }
+      return data
     },
     filteredLogData () {
       return this.logData.filter(d => {
@@ -79,9 +116,45 @@ export default {
           isAnsi
         }
       }
+    },
+    onToBottom () {
+      if (this.isScrolling) return
+      this.isToBottom = true
+    },
+    onScroll () {
+      if (this.isScrolling2nd) {
+        this.isToBottom = false
+      }
+      this.isScrolling = true
+      setTimeout(() => {
+        this.isScrolling2nd = true
+        setTimeout(() => {
+          this.isScrolling2nd = false
+        }, 50)
+      }, 50)
+      setTimeout(() => {
+        this.isScrolling = false
+      }, 100)
+    },
+    toBottom () {
+      this.$el.querySelector('.log-view').scrollTo({ top: 99999999 })
+      setTimeout(() => {
+        this.isToBottom = true
+      }, 50)
+    },
+    onAutoScroll () {
+      this.toBottom()
     }
   },
   mounted () {
+    this.handle = setInterval(() => {
+      if (this.isToBottom) {
+        this.toBottom()
+      }
+    }, 100)
+  },
+  destroyed () {
+    clearInterval(this.handle)
   }
 }
 </script>

frontend/src/i18n/zh.js

@@ -209,6 +209,8 @@ export default {
   'Random': '随机',
   'Selected Nodes': '指定节点',
   'Search Log': '搜索日志',
+  'Auto-Scroll': '自动滚动',
+  'Updating log...': '正在更新日志...',
 
   // 任务列表
   'Node': '节点',

jenkins/master/docker-compose.yaml

@@ -18,6 +18,8 @@ services:
     depends_on:
       - mongo
       - redis
+    volumes:
+      - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
   worker:
     image: "tikazyq/crawlab:master"
     environment:
@@ -30,6 +32,8 @@ services:
     depends_on:
       - mongo
       - redis
+    volumes:
+      - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
   mongo:
     image: mongo:latest
     restart: always