Merge pull request ynqa#41 from ynqa/repl

Add REPL

.travis.yml

@@ -26,7 +26,7 @@ script:
   - make lint
 
 after_script: |
-  if [[ $TRAVIS_GO_VERSION == 1.11* ]] && [[ "$TRAVIS_BRANCH" == "master" ]]; then
+  if [[ $TRAVIS_GO_VERSION == 1.11* ]] && [[ "$TRAVIS_BRANCH" == "master" ]] && [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
     goveralls -repotoken ${COVERALLS_TOKEN}
     echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
     docker build -t ynqa/wego:latest .

Gopkg.toml

@@ -25,3 +25,7 @@
 [[constraint]]
   name = "gopkg.in/cheggaaa/pb.v1"
   version = "1.0.18"
+
+[[constraint]]
+  name = "github.com/peterh/liner"
+  version = "1.1.0"

README.md

@@ -44,6 +44,7 @@ Usage:
 Available Commands:
   glove       GloVe: Global Vectors for Word Representation
   help        Help about any command
+  repl        Search similar words with REPL mode
   search      Search similar words
   word2vec    Word2Vec: Continuous Bag-of-Words and Skip-gram model
 
@@ -54,9 +55,10 @@ Use "wego [command] --help" for more information about a command.
 ```
 
 For more information about each sub-command, see below:
-- [search](./search/README.md)
 - [word2vec](./model/README.md)
 - [glove](./model/README.md)
+- [search](./search/README.md)
+- [repl](./repl/README.md)
 
 ## Demo
 

cmd/repl.go

@@ -0,0 +1,68 @@
+// Copyright © 2019 Makoto Ito
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/ynqa/wego/config"
+	"github.com/ynqa/wego/repl"
+)
+
+var ReplCmd = &cobra.Command{
+	Use:   "repl",
+	Short: "Search similar words with REPL mode",
+	Long:  "Search similar words with REPL mode",
+	Example: "  wego repl -i example/word_vectors.txt\n" +
+		"  >> apple + banana\n" +
+		"  ...",
+	PreRun: func(cmd *cobra.Command, args []string) {
+		replBind(cmd)
+	},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return executeRepl()
+	},
+}
+
+func init() {
+	ReplCmd.Flags().StringP(config.InputFile.String(), "i", config.DefaultOutputFile,
+		"input file path for trained word vector")
+	ReplCmd.Flags().IntP(config.Rank.String(), "r", config.DefaultRank,
+		"how many the most similar words will be displayed")
+}
+
+func replBind(cmd *cobra.Command) {
+	viper.BindPFlag(config.InputFile.String(), cmd.Flags().Lookup(config.InputFile.String()))
+	viper.BindPFlag(config.Rank.String(), cmd.Flags().Lookup(config.Rank.String()))
+}
+
+func executeRepl() error {
+	inputFile := viper.GetString(config.InputFile.String())
+	f, err := os.Open(inputFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	k := viper.GetInt(config.Rank.String())
+	repl, err := repl.NewRepl(f, k)
+	if err != nil {
+		return err
+	}
+	return repl.Run()
+}

cmd/repl_test.go

@@ -0,0 +1,34 @@
+// Copyright © 2019 Makoto Ito
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+)
+
+const replFlagSize = 2
+
+func TestReplBind(t *testing.T) {
+	defer viper.Reset()
+
+	replBind(ReplCmd)
+
+	if len(viper.AllKeys()) != replFlagSize {
+		t.Errorf("Expected replBind maps %v keys: %v",
+			replFlagSize, viper.AllKeys())
+	}
+}

cmd/root.go

@@ -78,4 +78,5 @@ func init() {
 	RootCmd.AddCommand(Word2vecCmd)
 	RootCmd.AddCommand(SearchCmd)
 	RootCmd.AddCommand(GloveCmd)
+	RootCmd.AddCommand(ReplCmd)
 }

cmd/search.go

@@ -43,7 +43,7 @@ var SearchCmd = &cobra.Command{
 }
 
 func init() {
-	SearchCmd.Flags().StringP(config.InputFile.String(), "i", config.DefaultInputFile,
+	SearchCmd.Flags().StringP(config.InputFile.String(), "i", config.DefaultOutputFile,
 		"input file path for trained word vector")
 	SearchCmd.Flags().IntP(config.Rank.String(), "r", config.DefaultRank,
 		"how many the most similar words will be displayed")
@@ -54,21 +54,21 @@ func searchBind(cmd *cobra.Command) {
 	viper.BindPFlag(config.InputFile.String(), cmd.Flags().Lookup(config.InputFile.String()))
 }
 
-func executeSearch(target string) error {
+func executeSearch(query string) error {
 	inputFile := viper.GetString(config.InputFile.String())
 	f, err := os.Open(inputFile)
 	if err != nil {
 		return err
 	}
+	defer f.Close()
 
-	parser := search.NewParser(f)
-	searcher, err := search.NewSearcher(parser)
+	searcher, err := search.NewSearcher(f)
 	if err != nil {
 		return err
 	}
 
 	k := viper.GetInt(config.Rank.String())
-	neighbors, err := searcher.Search(target, k)
+	neighbors, err := searcher.SearchWithQuery(query, k)
 	if err != nil {
 		return err
 	}

repl/README.md

@@ -0,0 +1,45 @@
+# REPL
+
+REPL mode for similarity search
+
+![wego](https://user-images.githubusercontent.com/6745370/51677211-2e54e700-201c-11e9-8ce9-19d4b84ef071.gif)
+
+## Usage
+
+```
+Search similar words with REPL mode
+
+Usage:
+  wego repl [flags]
+
+Examples:
+  wego repl -i example/word_vectors.txt
+  >> apple + banana
+  ...
+
+Flags:
+  -h, --help               help for repl
+  -i, --inputFile string   input file path for trained word vector (default "example/word_vectors.txt")
+  -r, --rank int           how many the most similar words will be displayed (default 10)
+```
+
+## Example
+
+Now, it is able to use `+`, `-` for arithmetic operations.
+
+```
+$ go run wego.go repl -i example/word_vectors_sg.txt
+>> a + b
+  RANK |  WORD   | SIMILARITY
++------+---------+------------+
+     1 | phi     |   0.907975
+     2 | q       |   0.904593
+     3 | mathbf  |   0.903066
+     4 | cdot    |   0.902205
+     5 | b       |   0.901952
+     6 | becomes |   0.900346
+     7 | int     |   0.898680
+     8 | z       |   0.897895
+     9 | named   |   0.896480
+    10 | v       |   0.895456
+```

repl/op.go

@@ -0,0 +1,40 @@
+// Copyright © 2019 Makoto Ito
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repl
+
+type Operator func(float64, float64) float64
+
+func elementWise(v1, v2 []float64, op Operator) []float64 {
+	for i := 0; i < len(v1); i++ {
+		v1[i] = op(v1[i], v2[i])
+	}
+	return v1
+}
+
+func addOp(x, y float64) float64 {
+	return x + y
+}
+
+func Add(v1, v2 []float64) []float64 {
+	return elementWise(v1, v2, addOp)
+}
+
+func subOp(x, y float64) float64 {
+	return x - y
+}
+
+func Sub(v1, v2 []float64) []float64 {
+	return elementWise(v1, v2, subOp)
+}