add tabnine golang server

.gitignore

@@ -4,3 +4,4 @@ build
 dist
 *.egg-info
 *.ipynb
+go/cmd/binaries

Dockerfile

@@ -1,15 +1,6 @@
-FROM python:3.7-alpine3.10
-
-ENV TABNINE_BINARY_PATH=/usr/local/tabnine/binary/TabNine
-RUN TAB_LINE_VERSION=$(wget -qO- https://update.tabnine.com/version) \
-    && mkdir -p /usr/local/tabnine/binary \
-    && wget -O ${TABNINE_BINARY_PATH} \
-        https://update.tabnine.com/${TAB_LINE_VERSION}/x86_64-unknown-linux-gnu/TabNine \
-        && chmod 777 ${TABNINE_BINARY_PATH}
-
-COPY server/server /usr/local/bin/server
-RUN chmod 777 /usr/local/bin/server
-ENV SERVER_PORT 8080
+FROM python:3.7-alpine3.11
+COPY go/cmd/server /usr/local/bin/tabnine-server
+RUN apk add build-base && pip install python-language-server
+RUN chmod 777 /usr/local/bin/tabnine-server
 EXPOSE 8080
-EXPOSE 5555
-ENTRYPOINT ["/usr/local/bin/server"]
+ENTRYPOINT ["/usr/local/bin/tabnine-server", "-libBaseDir=/usr/local/lib", "-port=8080"]

build-image.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if ! type "docker" > /dev/null; then
+    echo "Please install docker first!"
+fi
+
+wd=$(pwd)
+container_wd=${wd#${HOME}}
+echo "container working directory: ${container_wd}"
+docker run --rm -v ${HOME}/go:/go golang:1.14-alpine3.11 \
+    go build -o ${container_wd}/go/cmd/server ${container_wd}/go/cmd/server.go
+
+docker build -t tabnine-server:latest .

go/cmd/server.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/wenmin-wu/jupyter-tabnine/go/pkg/tabnine"
+)
+
+func main() {
+	var libBaseDir string
+
+	var port int
+
+	flag.StringVar(&libBaseDir, "libBaseDir", "./", "base directory of tabnine binaries")
+	flag.IntVar(&port, "port", 9999, "Server port")
+	flag.Parse()
+
+	tn, err := tabnine.NewTabNine(libBaseDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	http.HandleFunc("/tabnine", func(w http.ResponseWriter, r *http.Request) {
+		// fix cross-domain request problem
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		urlStr, _ := url.QueryUnescape(r.URL.String())
+		index := strings.Index(urlStr, "=")
+		data := []byte(urlStr[index+1:])
+		_, err = w.Write(tn.Request(data))
+	})
+
+	numSignals := 3
+	ch := make(chan os.Signal, numSignals)
+
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
+
+	go func() {
+		signalType := <-ch
+		signal.Stop(ch)
+		tn.Close()
+		log.Printf("Signal Type: %s\n", signalType)
+		os.Exit(0)
+	}()
+	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))
+}

go/pkg/tabnine/tabnine.go

@@ -0,0 +1,264 @@
+package tabnine
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/coreos/go-semver/semver"
+)
+
+type TabNine struct {
+	baseDir       string
+	cmd           *exec.Cmd
+	outReader     *bufio.Reader
+	mux           sync.Mutex
+	inPipeWriter  *io.PipeWriter
+	outPipeWriter *io.PipeWriter
+	inPipeReader  *io.PipeReader
+	outPipeReader *io.PipeReader
+	completeRes   *AutocompleteResult
+	emptyRes      []byte
+}
+
+type AutocompleteResult struct {
+	OldPrefix   string         `json:"old_prefix"`
+	Results     []*ResultEntry `json:"results"`
+	UserMessage []string       `json:"user_message"`
+}
+
+type ResultEntry struct {
+	NewPrefix string `json:"new_prefix"`
+	OldSuffix string `json:"old_suffix"`
+	NewSuffix string `json:"new_suffix"`
+	Details   string `json:"detail"`
+}
+
+const (
+	updateVersionUrl  = "https://update.tabnine.com/version"
+	downloadUrlPrefix = "https://update.tabnine.com"
+)
+
+var systemMap = map[string]string{
+	"darwin":  "apple-darwin",
+	"linux":   "unknown-linux-gnu",
+	"windows": "pc-windows-gnu",
+}
+
+func NewTabNine(baseDir string) (*TabNine, error) {
+	empty := AutocompleteResult{}
+	emptyRes, _ := json.Marshal(empty)
+	tabnine := &TabNine{
+		baseDir:     baseDir,
+		completeRes: &empty,
+		emptyRes:    emptyRes,
+	}
+	err := tabnine.init()
+	return tabnine, err
+}
+
+func (t *TabNine) init() (err error) {
+	log.Println("TabNine Initializing")
+	// download if needed
+	var binaryPath string
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func(wg *sync.WaitGroup) {
+		binaryPath, err = t.getBinaryPath()
+		wg.Done()
+	}(&wg)
+	t.inPipeReader, t.inPipeWriter = io.Pipe()
+	t.outPipeReader, t.outPipeWriter = io.Pipe()
+	wg.Wait()
+	if err == nil {
+		t.cmd = exec.Command(binaryPath, "--client=vscode")
+		t.cmd.Stdin = t.inPipeReader
+		t.cmd.Stdout = t.outPipeWriter
+		t.outReader = bufio.NewReader(t.outPipeReader)
+		err = t.cmd.Start()
+		// go t.cmd.Wait()
+	}
+	log.Println("TabNine Initialized")
+	return
+}
+
+func (t *TabNine) downloadBinary(url, binaryPath string) (err error) {
+	binaryDir := filepath.Dir(binaryPath)
+	isExist, isDir := checkDir(binaryDir)
+	if isExist && !isDir {
+		err = os.RemoveAll(binaryDir)
+		if err != nil {
+			return
+		}
+	}
+
+	if !isExist {
+		err = os.MkdirAll(binaryDir, os.ModePerm)
+		if err != nil {
+			return
+		}
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		err = fmt.Errorf("Request update version error: %s", resp.Status)
+		return
+	}
+	defer resp.Body.Close()
+
+	out, err := os.Create(binaryPath)
+	if err != nil {
+		return
+	}
+	defer out.Close()
+	_, err = io.Copy(out, resp.Body)
+	return
+}
+
+func (t *TabNine) getBinaryPath() (binaryPath string, err error) {
+	binaryDir := t.baseDir + "/binaries"
+	if err != nil {
+		return
+	}
+	needCreateDir := true
+	isExist, isDir := checkDir(binaryDir)
+	if isExist && isDir {
+		needCreateDir = false
+	}
+	if isExist && !isDir {
+		err = os.RemoveAll(binaryDir)
+		if err != nil {
+			return
+		}
+	}
+
+	if needCreateDir {
+		os.MkdirAll(binaryDir, os.ModePerm)
+	}
+
+	dirs, err := ioutil.ReadDir(binaryDir)
+	if err != nil {
+		return
+	}
+
+	var versions []*semver.Version
+
+	for _, d := range dirs {
+		versions = append(versions, semver.New(d.Name()))
+	}
+	semver.Sort(versions)
+	arch := parseArch(runtime.GOARCH)
+	sys := systemMap[strings.ToLower(runtime.GOOS)]
+	exeName := "TabNine"
+	if strings.ToLower(runtime.GOOS) == "windows" {
+		exeName += ".exe"
+	}
+	triple := fmt.Sprintf("%s-%s", arch, sys)
+	for _, v := range versions {
+		binaryPath = filepath.Join(binaryDir, v.String(), triple, exeName)
+		if isFile(binaryPath) {
+			err = os.Chmod(binaryPath, 0755)
+			return
+		}
+	}
+	// need download
+	resp, err := http.Get(updateVersionUrl)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		err = fmt.Errorf("Request update version error: %s", resp.Status)
+		return
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	log.Println("Binary doesn't exist, starting download.'")
+	latestVersion := strings.TrimSpace(string(body))
+	log.Printf("Latest version: %s\n", latestVersion)
+	subPath := filepath.Join(latestVersion, triple, exeName)
+	binaryPath = filepath.Join(binaryDir, subPath)
+	downloadUrl := fmt.Sprintf("%s/%s", downloadUrlPrefix, subPath)
+	log.Printf("Download url: %s, Binary path: %s", downloadUrl, binaryPath)
+	err = t.downloadBinary(downloadUrl, binaryPath)
+	if err != nil {
+		log.Fatal("Download failed ", err)
+		return
+	}
+	err = os.Chmod(binaryPath, 0755)
+	log.Println("Download finished.")
+	return
+}
+
+func (t *TabNine) Request(data []byte) (res []byte) {
+	t.mux.Lock()
+	t.inPipeWriter.Write(data)
+	t.inPipeWriter.Write([]byte("\n"))
+	bytes, err := t.outReader.ReadBytes('\n')
+	t.mux.Unlock()
+	if err != nil {
+		res = t.emptyRes
+		return
+	}
+	// remove useless fields
+	err = json.Unmarshal(bytes, t.completeRes)
+	if err != nil {
+		res = t.emptyRes
+		return
+	}
+	res, err = json.Marshal(t.completeRes)
+	return
+}
+
+func (t *TabNine) Close() {
+	log.Println("tabnine closing... cleaning up...")
+	t.cmd.Process.Kill()
+	t.inPipeWriter.Close()
+	t.outPipeWriter.Close()
+	t.inPipeReader.Close()
+	t.outPipeReader.Close()
+}
+
+func checkDir(path string) (isExist, isDir bool) {
+	info, err := os.Stat(path)
+	isExist = false
+	if os.IsNotExist(err) {
+		return
+	}
+	isExist = true
+	isDir = info.IsDir()
+	return
+}
+
+func isFile(path string) bool {
+	isExist, isDir := checkDir(path)
+	return isExist && !isDir
+}
+
+func isDir(path string) bool {
+	isExist, isDir := checkDir(path)
+	return isExist && isDir
+}
+
+func parseArch(arch string) string {
+	if strings.ToLower(arch) == "amd64" {
+		return "x86_64"
+	}
+	return arch
+}

src/jupyter_tabnine/static/main.js

@@ -118,6 +118,9 @@ define([
         // use get to solve post redirecting too many times
         $.get(serverUrl, { 'data': JSON.stringify(requestData) })
             .done(function (data) {
+                if (typeof data === 'string') {
+                    data = JSON.parse(data);
+                }
                 handleResData(data);
             }).fail(function (error) {
                 console.log(logPrefix, ' get error: ', error);