refactor: finish first pass of dataset.Transform refactor

Merge pull request #164 from qri-io/refactor/ds_transform

Dockerfile

@@ -4,10 +4,10 @@ LABEL maintainer="[email protected]"
 ADD . /go/src/github.com/qri-io/qri
 RUN cd /go/src/github.com/qri-io/qri
 
-# RUN go get -u github.com/whyrusleeping/gx
-# RUN go get -u github.com/whyrusleeping/gx-go
-# RUN gx install
-# RUN go get ./...
+RUN go get -v github.com/briandowns/spinner github.com/datatogether/api/apiutil github.com/fatih/color github.com/ipfs/go-datastore github.com/olekukonko/tablewriter github.com/qri-io/analytics github.com/qri-io/bleve github.com/qri-io/dataset github.com/qri-io/dataset_sql github.com/qri-io/doggos github.com/sirupsen/logrus github.com/spf13/cobra github.com/spf13/viper
+
+RUN go get -u github.com/whyrusleeping/gx github.com/whyrusleeping/gx-go
+RUN cd /go/src/github.com/qri-io/qri && pwd && gx install
 
 RUN go install github.com/qri-io/qri
 # set default port to 8080, default log level, QRI_PATH env, IPFS_PATH env

api/config.go

@@ -11,14 +11,17 @@ const (
 	DEVELOP_MODE    = "develop"
 	PRODUCTION_MODE = "production"
 	TEST_MODE       = "test"
+	DefaultPort     = "2503"
+	DefaultRPCPort  = "2504"
 )
 
 func DefaultConfig() *Config {
 	return &Config{
-		Logger: logging.DefaultLogger,
-		Mode:   "develop",
-		Port:   "8080",
-		Online: true,
+		Logger:  logging.DefaultLogger,
+		Mode:    "develop",
+		Port:    DefaultPort,
+		RPCPort: DefaultRPCPort,
+		Online:  true,
 	}
 }
 
@@ -38,6 +41,8 @@ type Config struct {
 	Mode string
 	// port to listen on, will be read from PORT env variable if present.
 	Port string
+	// port to listen for RPC calls on, if empty server will not register a RPC listener
+	RPCPort string
 	// root url for service
 	UrlRoot string
 	// DNS service discovery. Should be either "env" or "dns", default is env
@@ -67,7 +72,7 @@ type Config struct {
 func (cfg *Config) Validate() (err error) {
 	// make sure port is set
 	if cfg.Port == "" {
-		cfg.Port = "8080"
+		cfg.Port = DefaultPort
 	}
 
 	err = requireConfigStrings(map[string]string{

api/handlers/datasets.go

@@ -17,7 +17,7 @@ import (
 )
 
 func NewDatasetHandlers(log logging.Logger, r repo.Repo) *DatasetHandlers {
-	req := core.NewDatasetRequests(r)
+	req := core.NewDatasetRequests(r, nil)
 	h := DatasetHandlers{*req, log, r}
 	return &h
 }

api/handlers/history.go

@@ -17,7 +17,7 @@ type HistoryHandlers struct {
 }
 
 func NewHistoryHandlers(log logging.Logger, r repo.Repo) *HistoryHandlers {
-	req := core.NewHistoryRequests(r)
+	req := core.NewHistoryRequests(r, nil)
 	h := HistoryHandlers{*req, log}
 	return &h
 }

api/handlers/peers.go

@@ -14,7 +14,7 @@ import (
 )
 
 func NewPeerHandlers(log logging.Logger, r repo.Repo, node *p2p.QriNode) *PeerHandlers {
-	req := core.NewPeerRequests(r, node)
+	req := core.NewPeerRequests(node, nil)
 	h := PeerHandlers{*req, log}
 	return &h
 }

api/handlers/profile.go

@@ -8,7 +8,6 @@ import (
 	"github.com/qri-io/qri/core"
 	"github.com/qri-io/qri/logging"
 	"github.com/qri-io/qri/repo"
-	"github.com/qri-io/qri/repo/profile"
 )
 
 // ProfileHandlers wraps a requests struct to interface with http.HandlerFunc
@@ -18,7 +17,7 @@ type ProfileHandlers struct {
 }
 
 func NewProfileHandlers(log logging.Logger, r repo.Repo) *ProfileHandlers {
-	req := core.NewProfileRequests(r)
+	req := core.NewProfileRequests(r, nil)
 	h := ProfileHandlers{*req, log}
 	return &h
 }
@@ -38,7 +37,7 @@ func (h *ProfileHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 
 func (h *ProfileHandlers) getProfileHandler(w http.ResponseWriter, r *http.Request) {
 	args := true
-	res := &profile.Profile{}
+	res := &core.Profile{}
 	if err := h.GetProfile(&args, res); err != nil {
 		h.log.Infof("error getting profile: %s", err.Error())
 		util.WriteErrResponse(w, http.StatusInternalServerError, err)
@@ -49,12 +48,12 @@ func (h *ProfileHandlers) getProfileHandler(w http.ResponseWriter, r *http.Reque
 }
 
 func (h *ProfileHandlers) saveProfileHandler(w http.ResponseWriter, r *http.Request) {
-	p := &profile.Profile{}
+	p := &core.Profile{}
 	if err := json.NewDecoder(r.Body).Decode(p); err != nil {
 		util.WriteErrResponse(w, http.StatusBadRequest, err)
 		return
 	}
-	res := &profile.Profile{}
+	res := &core.Profile{}
 	if err := h.SaveProfile(p, res); err != nil {
 		util.WriteErrResponse(w, http.StatusInternalServerError, err)
 		return
@@ -91,7 +90,7 @@ func (h *ProfileHandlers) setProfilePhotoHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	res := &profile.Profile{}
+	res := &core.Profile{}
 	if err := h.SetProfilePhoto(p, res); err != nil {
 		h.log.Infof("error initializing dataset: %s", err.Error())
 		util.WriteErrResponse(w, http.StatusInternalServerError, err)
@@ -128,7 +127,7 @@ func (h *ProfileHandlers) setPosterHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	res := &profile.Profile{}
+	res := &core.Profile{}
 	if err := h.SetPosterPhoto(p, res); err != nil {
 		h.log.Infof("error initializing dataset: %s", err.Error())
 		util.WriteErrResponse(w, http.StatusInternalServerError, err)

api/handlers/queries.go

@@ -12,7 +12,7 @@ import (
 )
 
 func NewQueryHandlers(log logging.Logger, r repo.Repo) *QueryHandlers {
-	req := core.NewQueryRequests(r)
+	req := core.NewQueryRequests(r, nil)
 	return &QueryHandlers{*req, log}
 }
 

api/handlers/search.go

@@ -17,7 +17,7 @@ type SearchHandlers struct {
 }
 
 func NewSearchHandlers(log logging.Logger, r repo.Repo) *SearchHandlers {
-	req := core.NewSearchRequests(r)
+	req := core.NewSearchRequests(r, nil)
 	return &SearchHandlers{*req, log}
 }
 

api/server.go

@@ -2,10 +2,13 @@ package api
 
 import (
 	"fmt"
+	"net"
 	"net/http"
+	"net/rpc"
 
 	"github.com/datatogether/api/apiutil"
 	"github.com/qri-io/qri/api/handlers"
+	"github.com/qri-io/qri/core"
 	"github.com/qri-io/qri/logging"
 	"github.com/qri-io/qri/p2p"
 	"github.com/qri-io/qri/repo"
@@ -71,10 +74,35 @@ func (s *Server) Serve() (err error) {
 	server := &http.Server{}
 	server.Handler = NewServerRoutes(s)
 	s.log.Infof("starting api server on port %s", s.cfg.Port)
+	go s.ServeRPC()
 	// http.ListenAndServe will not return unless there's an error
 	return StartServer(s.cfg, server)
 }
 
+// ServeRPC checks for a configured RPC port, and registers a listner if so
+func (s *Server) ServeRPC() {
+	if s.cfg.RPCPort == "" {
+		return
+	}
+
+	listener, err := net.Listen("tcp", fmt.Sprintf(":%s", s.cfg.RPCPort))
+	if err != nil {
+		s.log.Infof("RPC listen on port %s error: %s", s.cfg.RPCPort, err)
+		return
+	}
+
+	for _, rcvr := range core.Receivers(s.qriNode) {
+		if err := rpc.Register(rcvr); err != nil {
+			s.log.Infof("error registering RPC receiver %s: %s", rcvr.CoreRequestsName(), err.Error())
+			return
+		}
+	}
+
+	s.log.Infof("accepting RPC requests on port %s", s.cfg.RPCPort)
+	rpc.Accept(listener)
+	return
+}
+
 // NewServerRoutes returns a Muxer that has all API routes
 func NewServerRoutes(s *Server) *http.ServeMux {
 	m := http.NewServeMux()

cmd/add.go

@@ -34,15 +34,16 @@ var datasetAddCmd = &cobra.Command{
 				ErrExit(fmt.Errorf("please provide a --name"))
 			}
 
-			req := core.NewDatasetRequests(GetRepo(false))
+			req, err := DatasetRequests(false)
+			ExitIfErr(err)
 
 			root := strings.TrimSuffix(args[0], "/"+dsfs.PackageFileDataset.String())
 			p := &core.AddParams{
 				Name: addDsName,
 				Hash: root,
 			}
 			res := &repo.DatasetRef{}
-			err := req.AddDataset(p, res)
+			err = req.AddDataset(p, res)
 			ExitIfErr(err)
 
 			PrintInfo("Successfully added dataset %s: %s", addDsName, res.Path.String())
@@ -69,7 +70,8 @@ func initDataset() {
 	metaFile, err = loadFileIfPath(addDsMetaFilepath)
 	ExitIfErr(err)
 
-	req := core.NewDatasetRequests(GetRepo(false))
+	req, err := DatasetRequests(false)
+	ExitIfErr(err)
 
 	p := &core.InitDatasetParams{
 		Name:         addDsName,

cmd/cmd.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
 
+	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/viper"
 )
 
@@ -41,14 +41,20 @@ func cachePath() string {
 }
 
 func userHomeDir() string {
-	if runtime.GOOS == "windows" {
-		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
-		if home == "" {
-			home = os.Getenv("USERPROFILE")
-		}
-		return home
+	dir, err := homedir.Dir()
+	if err != nil {
+		panic(err)
 	}
-	return os.Getenv("HOME")
+	return dir
+
+	// if runtime.GOOS == "windows" {
+	// 	home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+	// 	if home == "" {
+	// 		home = os.Getenv("USERPROFILE")
+	// 	}
+	// 	return home
+	// }
+	// return os.Getenv("HOME")
 }
 
 func loadFileIfPath(path string) (file *os.File, err error) {

cmd/config.go

@@ -51,7 +51,8 @@ func init() {
 }
 
 // initConfig reads in config file and ENV variables if set.
-func initConfig() {
+func initConfig() (created bool) {
+	var err error
 	home := userHomeDir()
 	SetNoColor()
 
@@ -76,19 +77,21 @@ func initConfig() {
 
 	ipfsFsPath := os.Getenv("IPFS_PATH")
 	if ipfsFsPath == "" {
-		ipfsFsPath = "$HOME/.ipfs"
+		ipfsFsPath = filepath.Join(home, ".ipfs")
 	}
 	ipfsFsPath = strings.Replace(ipfsFsPath, "~", home, 1)
 	viper.SetDefault(IpfsFsPath, ipfsFsPath)
 
 	viper.SetConfigName("config") // name of config file (without extension)
 	viper.AddConfigPath(qriPath)  // add QRI_PATH env var
 
-	err := EnsureConfigFile()
+	created, err = EnsureConfigFile()
 	ExitIfErr(err)
 
 	err = viper.ReadInConfig()
 	ExitIfErr(err)
+
+	return
 }
 
 func configFilepath() string {
@@ -99,12 +102,12 @@ func configFilepath() string {
 	return path
 }
 
-func EnsureConfigFile() error {
+func EnsureConfigFile() (bool, error) {
 	if _, err := os.Stat(configFilepath()); os.IsNotExist(err) {
 		fmt.Println("writing config file")
-		return WriteConfigFile(defaultCfg)
+		return true, WriteConfigFile(defaultCfg)
 	}
-	return nil
+	return false, nil
 }
 
 func WriteConfigFile(cfg *Config) error {

cmd/export.go

@@ -30,7 +30,7 @@ var exportCmd = &cobra.Command{
 		}
 
 		r := GetRepo(false)
-		req := core.NewDatasetRequests(r)
+		req := core.NewDatasetRequests(r, nil)
 
 		p := &core.GetDatasetParams{
 			Name: args[0],

cmd/info.go

@@ -1,9 +1,11 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/ipfs/go-datastore"
+	"github.com/qri-io/dataset"
 	"github.com/qri-io/dataset/dsfs"
 	"github.com/qri-io/qri/core"
 	"github.com/qri-io/qri/repo"
@@ -21,7 +23,19 @@ var infoCmd = &cobra.Command{
 			ErrExit(fmt.Errorf("please specify a dataset path or name to get the info of"))
 		}
 
-		req := core.NewDatasetRequests(GetRepo(false))
+		outformat := cmd.Flag("format").Value.String()
+		if outformat != "" {
+			format, err := dataset.ParseDataFormatString(outformat)
+			if err != nil {
+				ErrExit(fmt.Errorf("invalid data format: %s", cmd.Flag("format").Value.String()))
+			}
+			if format != dataset.JSONDataFormat {
+				ErrExit(fmt.Errorf("invalid data format. currently only json or plaintext are supported"))
+			}
+		}
+
+		req, err := DatasetRequests(false)
+		ExitIfErr(err)
 
 		for i, arg := range args {
 			rt, ref := dsfs.RefType(arg)
@@ -35,11 +49,18 @@ var infoCmd = &cobra.Command{
 			res := &repo.DatasetRef{}
 			err := req.Get(p, res)
 			ExitIfErr(err)
-			PrintDatasetRefInfo(i, res)
+			if outformat == "" {
+				PrintDatasetRefInfo(i, res)
+			} else {
+				data, err := json.MarshalIndent(res.Dataset, "", "  ")
+				ExitIfErr(err)
+				fmt.Printf("%s", string(data))
+			}
 		}
 	},
 }
 
 func init() {
 	RootCmd.AddCommand(infoCmd)
+	infoCmd.Flags().StringP("format", "f", "", "set output format [json]")
 }