Merge branch 'develop' of https://github.com/crawlab-team/crawlab int…

…o develop

backend/scripts/install-nodejs.sh

@@ -10,9 +10,8 @@ touch /tmp/install.lock
 touch /tmp/install-nodejs.lock
 
 # install node.js
-curl -sL https://deb.nodesource.com/setup_10.x | bash -
-apt-get update && apt install -y nodejs nodejs-dev node-gyp libssl1.0-dev
-apt-get update && apt install -y npm
+curl -sL https://deb.nodesource.com/setup_12.x | bash -
+apt-get update && apt install -y nodejs 
 
 # install chromium
 # See https://crbug.com/795759

backend/services/sys_exec/linux_mac.go

@@ -0,0 +1,30 @@
+// +build !windows
+
+package sys_exec
+
+import (
+ "os/exec"
+ "syscall"
+)
+
+func BuildCmd(cmdStr string) *exec.Cmd {
+ return exec.Command("sh", "-c", cmdStr)
+}
+
+func Setpgid(cmd *exec.Cmd) {
+ if cmd == nil {
+ return
+ }
+ if cmd.SysProcAttr == nil {
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ } else {
+ cmd.SysProcAttr.Setpgid = true
+ }
+}
+
+func KillProcess(cmd *exec.Cmd) error {
+ if cmd == nil {
+ return nil
+ }
+ return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+}

backend/services/sys_exec/windows.go

@@ -0,0 +1,24 @@
+// +build windows
+
+package sys_exec
+
+import (
+ "os/exec"
+)
+
+func BuildCmd(cmdStr string) *exec.Cmd {
+ return exec.Command("cmd", "/C", cmdStr)
+}
+
+func Setpgid(cmd *exec.Cmd) {
+ return
+}
+
+func KillProcess(cmd *exec.Cmd) error {
+ if cmd != nil && cmd.Process != nil {
+ if err := cmd.Process.Kill(); err != nil {
+ return err
+ }
+ }
+ return nil
+}

backend/services/task.go

@@ -10,6 +10,7 @@ import (
  "crawlab/services/notification"
  "crawlab/services/rpc"
  "crawlab/services/spider_handler"
+ "crawlab/services/sys_exec"
  "crawlab/utils"
  "encoding/json"
  "errors"
@@ -24,12 +25,10 @@ import (
  "os"
  "os/exec"
  "path/filepath"
- "runtime"
  "runtime/debug"
  "strconv"
  "strings"
  "sync"
- "syscall"
  "time"
 )
 
@@ -281,15 +280,8 @@ func FinishOrCancelTask(ch chan string, cmd *exec.Cmd, s model.Spider, t model.T
  log.Infof("process received signal: %s", signal)
 
  if signal == constants.TaskCancel && cmd.Process != nil {
- var err error
- // 兼容windows
- if runtime.GOOS == constants.Windows {
- err = cmd.Process.Kill()
- } else {
- err = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
- }
- // 取消进程
- if err != nil {
+ // 终止进程
+ if err := sys_exec.KillProcess(cmd); err != nil {
  log.Errorf("process kill error: %s", err.Error())
  debug.PrintStack()
 
@@ -359,12 +351,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
  wg := &sync.WaitGroup{}
 
  // 生成执行命令
- var cmd *exec.Cmd
- if runtime.GOOS == constants.Windows {
- cmd = exec.Command("cmd", "/C", cmdStr)
- } else {
- cmd = exec.Command("sh", "-c", cmdStr)
- }
+ cmd := sys_exec.BuildCmd(cmdStr)
 
  // 工作目录
  cmd.Dir = cwd
@@ -395,9 +382,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
  go FinishOrCancelTask(ch, cmd, s, t)
 
  // kill的时候，可以kill所有的子进程
- if runtime.GOOS != constants.Windows {
- cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
- }
+ sys_exec.Setpgid(cmd)
 
  // 启动进程
  if err := StartTaskProcess(cmd, t); err != nil {

docker_init.sh

@@ -22,6 +22,9 @@ fi
 # start nginx
 service nginx start
 
+#grant script 
+chmod +x /app/backend/scripts/*.sh
+
 # install languages
 if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_JAVA}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_DOTNET}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_PHP}" = "Y" ] || [ "${CRAWLAB_SERVER_LANG_GO}" = "Y" ];
 then