From 83e2fd3a86ea80ad720c53bbf27bb9a27810a1bf Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Wed, 6 Dec 2017 19:06:08 -0500 Subject: [PATCH 01/10] refactor: adapting qri to dataset.Transform refactor NOT READY FOR MERGE. initial round of updates to make qri work with new dataset.Transform, lots more to do. Also, added some -f flags to spit out json from the command line for tom. fixes #111 --- cmd/info.go | 22 +++++- cmd/list.go | 22 +++++- cmd/run.go | 7 +- core/datasets.go | 18 +++-- core/datasets_test.go | 3 + core/queries.go | 67 +++++++++++++----- core/queries_test.go | 151 +++++++++++++++++++++++------------------ p2p/record.go | 2 +- repo/dataset_ref.go | 2 +- repo/graph.go | 41 +++++------ repo/graph_test.go | 14 ++-- repo/test/test_repo.go | 2 +- 12 files changed, 223 insertions(+), 128 deletions(-) diff --git a/cmd/info.go b/cmd/info.go index 2d806af1e..1a35cf346 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -1,9 +1,11 @@ package cmd import ( + "encoding/json" "fmt" "github.com/ipfs/go-datastore" + "github.com/qri-io/dataset" "github.com/qri-io/dataset/dsfs" "github.com/qri-io/qri/core" "github.com/qri-io/qri/repo" @@ -21,6 +23,17 @@ var infoCmd = &cobra.Command{ ErrExit(fmt.Errorf("please specify a dataset path or name to get the info of")) } + outformat := cmd.Flag("format").Value.String() + if outformat != "" { + format, err := dataset.ParseDataFormatString(outformat) + if err != nil { + ErrExit(fmt.Errorf("invalid data format: %s", cmd.Flag("format").Value.String())) + } + if format != dataset.JSONDataFormat { + ErrExit(fmt.Errorf("invalid data format. currently only json or plaintext are supported")) + } + } + req := core.NewDatasetRequests(GetRepo(false)) for i, arg := range args { @@ -35,11 +48,18 @@ var infoCmd = &cobra.Command{ res := &repo.DatasetRef{} err := req.Get(p, res) ExitIfErr(err) - PrintDatasetRefInfo(i, res) + if outformat == "" { + PrintDatasetRefInfo(i, res) + } else { + data, err := json.MarshalIndent(res.Dataset, "", " ") + ExitIfErr(err) + fmt.Printf("%s", string(data)) + } } }, } func init() { RootCmd.AddCommand(infoCmd) + infoCmd.Flags().StringP("format", "f", "", "set output format [json]") } diff --git a/cmd/list.go b/cmd/list.go index 2b38be61d..26336102c 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -1,6 +1,9 @@ package cmd import ( + "encoding/json" + "fmt" + "github.com/qri-io/dataset" "github.com/spf13/cobra" ) @@ -10,14 +13,29 @@ var datasetListCmd = &cobra.Command{ Short: "list your local datasets", Long: ``, Run: func(cmd *cobra.Command, args []string) { + // TODO - add limit & offset params refs, err := GetRepo(false).Namespace(100, 0) ExitIfErr(err) - for _, ref := range refs { - PrintInfo("%s\t\t\t: %s", ref.Name, ref.Path) + + outformat := cmd.Flag("format").Value.String() + + switch outformat { + case "": + for _, ref := range refs { + PrintInfo("%s\t\t\t: %s", ref.Name, ref.Path) + } + case dataset.JSONDataFormat.String(): + data, err := json.MarshalIndent(refs, "", " ") + ExitIfErr(err) + fmt.Printf("%s\n", string(data)) + default: + ErrExit(fmt.Errorf("unrecognized format: %s", outformat)) } + }, } func init() { RootCmd.AddCommand(datasetListCmd) + datasetListCmd.Flags().StringP("format", "f", "", "set output format [json]") } diff --git a/cmd/run.go b/cmd/run.go index f4166c8f4..09cdc11d1 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -42,12 +42,9 @@ var runCmd = &cobra.Command{ SaveName: runCmdName, Dataset: &dataset.Dataset{ Timestamp: time.Now().In(time.UTC), - Query: &dataset.Query{ + Transform: &dataset.Transform{ Syntax: "sql", - Abstract: &dataset.AbstractQuery{ - Syntax: "sql", - Statement: args[0], - }, + Data: args[0], }, }, } diff --git a/core/datasets.go b/core/datasets.go index ba7e306c0..114370ba0 100644 --- a/core/datasets.go +++ b/core/datasets.go @@ -110,10 +110,11 @@ type InitDatasetParams struct { // InitDataset creates a new qri dataset from a source of data func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef) error { var ( - rdr io.Reader - store = r.repo.Store() + rdr io.Reader + store = r.repo.Store() + filename = p.DataFilename ) - var filename = p.DataFilename + if p.Url != "" { res, err := http.Get(p.Url) if err != nil { @@ -128,6 +129,12 @@ func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef return fmt.Errorf("either a file or a url is required to create a dataset") } + if p.Name != "" { + if err := validate.ValidName(p.Name); err != nil { + return fmt.Errorf("invalid name: %s", err.Error()) + } + } + // TODO - need a better strategy for huge files data, err := ioutil.ReadAll(rdr) if err != nil { @@ -194,7 +201,7 @@ func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef if ds.Title == "" { ds.Title = name } - ds.Data = datakey + ds.Data = datakey.String() if ds.Structure == nil { ds.Structure = &dataset.Structure{} } @@ -213,7 +220,6 @@ func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef return fmt.Errorf("error putting dataset in repo: %s", err.Error()) } - fmt.Println(dskey.String()) if err = r.repo.PutName(name, dskey); err != nil { return fmt.Errorf("error adding dataset name to repo: %s", err.Error()) } @@ -281,7 +287,7 @@ func (r *DatasetRequests) Update(p *UpdateParams, res *repo.DatasetRef) (err err return fmt.Errorf("error putting data in store: %s", err.Error()) } - ds.Data = path + ds.Data = path.String() ds.Length = len(data) } diff --git a/core/datasets_test.go b/core/datasets_test.go index 394f8e029..821d3c445 100644 --- a/core/datasets_test.go +++ b/core/datasets_test.go @@ -34,6 +34,9 @@ func TestDatasetRequestsInit(t *testing.T) { // Ensure that structure validation is being called {&InitDatasetParams{DataFilename: badStructureFile.FileName(), Data: badStructureFile}, nil, "invalid structure: error: cannot use the same name, 'colb' more than once"}, + // should reject invalid names + {&InitDatasetParams{DataFilename: jobsByAutomationFile.FileName(), Name: "foo bar baz", Data: jobsByAutomationFile}, nil, + "invalid name: error: illegal name 'foo bar baz', names must start with a letter and consist of only a-z,0-9, and _. max length 144 characters"}, // this should work {&InitDatasetParams{DataFilename: jobsByAutomationFile.FileName(), Data: jobsByAutomationFile}, nil, ""}, // Ensure that we can't double-add data diff --git a/core/queries.go b/core/queries.go index 2c446bc19..f350ff390 100644 --- a/core/queries.go +++ b/core/queries.go @@ -75,7 +75,7 @@ type RunParams struct { func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { var ( store = r.repo.Store() - structure *dataset.Structure + transform *dataset.Transform results []byte err error ds = p.Dataset @@ -88,19 +88,17 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { ds.Timestamp = time.Now() - q := ds.Query + q := ds.Transform if q == nil { - q = &dataset.Query{ + q = &dataset.Transform{ Syntax: "sql", - Abstract: &dataset.AbstractQuery{ - Syntax: "sql", - Statement: ds.QueryString, - }, + Data: ds.QueryString, } } - if ds.QueryString == "" { - ds.QueryString = q.Abstract.Statement - } + + // if ds.QueryString == "" { + // ds.QueryString = q.Abstract.Statement + // } // TODO - make format output the parsed statement as well // to avoid triple-parsing @@ -108,7 +106,7 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { // if err != nil { // return err // } - names, err := sql.StatementTableNames(q.Abstract.Statement) + names, err := sql.StatementTableNames(q.Data) if err != nil { return fmt.Errorf("error getting statement table names: %s", err.Error()) } @@ -129,11 +127,38 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { } } - qpath, err := sql.PreparedQueryPath(r.repo.Store(), q, &sql.ExecOpt{Format: dataset.CSVDataFormat}) + // func PreparedQueryPath(fs cafs.Filestore, q *dataset.Query, opts *ExecOpt) (datastore.Key, error) { + // q2 := &dataset.Query{} + // q2.Assign(q) + // prep, err := Prepare(q2, opts) + // if err != nil { + // return datastore.NewKey(""), err + // } + // return dsfs.SaveQuery(fs, prep.q, false) + // } + + q2 := &dataset.Transform{} + q2.Assign(q) + _, abst, err := sql.Format(q, func(o *sql.ExecOpt) { + o.Format = dataset.CSVDataFormat + }) + if err != nil { + return fmt.Errorf("formatting error: %s", err.Error()) + } + qpath, err := dsfs.SaveTransform(store, abst, false) if err != nil { return fmt.Errorf("error calculating query hash: %s", err.Error()) } + // fmt.Println(qpath.String()) + // atb, _ := abst.MarshalJSON() + // fmt.Println(string(atb)) + + // qpath, err := sql.PreparedQueryPath(r.repo.Store(), q, &sql.ExecOpt{Format: dataset.CSVDataFormat}) + // if err != nil { + // return fmt.Errorf("error calculating query hash: %s", err.Error()) + // } + if dsp, err := repo.DatasetForQuery(r.repo, qpath); err != nil && err != repo.ErrNotFound { return fmt.Errorf("error checking for existing query: %s", err.Error()) } else if err != repo.ErrNotFound { @@ -148,22 +173,27 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { } // TODO - detect data format from passed-in results structure - structure, results, err = sql.Exec(store, q, func(o *sql.ExecOpt) { + transform, results, err = sql.Exec(store, q, func(o *sql.ExecOpt) { o.Format = dataset.CSVDataFormat }) if err != nil { return fmt.Errorf("error executing query: %s", err.Error()) } + // tb, _ := transform.MarshalJSON() + // fmt.Println(string(tb)) + // TODO - move this into setting on the dataset outparam - ds.Structure = structure + ds.Structure = transform.Structure ds.Length = len(results) - ds.Query = q + ds.Transform = q + ds.AbstractTransform = transform - ds.Data, err = store.Put(memfs.NewMemfileBytes("data."+ds.Structure.Format.String(), results), false) + datakey, err := store.Put(memfs.NewMemfileBytes("data."+ds.Structure.Format.String(), results), false) if err != nil { return fmt.Errorf("error putting results in store: %s", err.Error()) } + ds.Data = datakey.String() pin := p.SaveName != "" @@ -181,10 +211,11 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { if err := dsfs.DerefDatasetStructure(store, ds); err != nil { return fmt.Errorf("error dereferencing dataset structure: %s", err.Error()) } - fmt.Println("result query:", ds.Query.Path()) - if err := dsfs.DerefDatasetQuery(store, ds); err != nil { + // fmt.Println("result query:", ds.AbstractTransform.Path()) + if err := dsfs.DerefDatasetTransform(store, ds); err != nil { return fmt.Errorf("error dereferencing dataset query: %s", err.Error()) } + fmt.Println(ds.AbstractTransform.Path().String()) ref := &repo.DatasetRef{Name: p.SaveName, Path: dspath, Dataset: ds} diff --git a/core/queries_test.go b/core/queries_test.go index 1de04dcb7..aeea58a99 100644 --- a/core/queries_test.go +++ b/core/queries_test.go @@ -1,6 +1,7 @@ package core import ( + "encoding/json" "fmt" "testing" @@ -10,19 +11,6 @@ import ( testrepo "github.com/qri-io/qri/repo/test" ) -// func TestNewQueryRequests(t *testing.T) { -// mr, err := testrepo.NewTestRepo() -// if err != nil { -// t.Errorf("error allocating test repo: %s", err.Error()) -// return -// } -// req := NewQueryRequests(mr) -// if req == nil { -// t.Errorf("error: expected non-nil result from NewQueryRequests()") -// return -// } -// } - func TestList(t *testing.T) { mr, err := testrepo.NewTestRepo() if err != nil { @@ -34,6 +22,7 @@ func TestList(t *testing.T) { t.Errorf("error: expected non-nil result from NewQueryRequests()") return } + cases := []struct { p *ListParams res *[]*repo.DatasetRef @@ -117,66 +106,96 @@ func TestRun(t *testing.T) { t.Errorf("case %d error mismatch: expected: %s, got: %s", i, c.err, err) continue } + + if c.err == "" { + fmt.Println("path:", got.Path.String()) + df, err := mr.Store().Get(got.Path) + if err != nil { + t.Errorf("case %d error getting dataset path: %s: %s", i, got.Path.String(), err.Error()) + continue + } + + ds := &dataset.Dataset{} + if err := json.NewDecoder(df).Decode(ds); err != nil { + t.Errorf("case %d decode dataset error: %s", i, err.Error()) + continue + } + + if !ds.Transform.IsEmpty() { + t.Errorf("expected stored dataset.Transform to be a reference") + } + if !ds.AbstractTransform.IsEmpty() { + t.Errorf("expected stored dataset.AbstractTransform to be a reference") + } + if !ds.Structure.IsEmpty() { + t.Errorf("expected stored dataset.Structure to be a reference") + } + if !ds.AbstractStructure.IsEmpty() { + t.Errorf("expected stored dataset.AbstractStructure to be a reference") + } + + } } } -func TestDatasetQueries(t *testing.T) { - mr, err := testrepo.NewTestRepo() - if err != nil { - t.Errorf("error allocating test repo: %s", err.Error()) - return - } +// TODO - RESTORE BEFORE MERGING +// func TestDatasetQueries(t *testing.T) { +// mr, err := testrepo.NewTestRepo() +// if err != nil { +// t.Errorf("error allocating test repo: %s", err.Error()) +// return +// } - req := NewQueryRequests(mr) +// req := NewQueryRequests(mr) - path, err := mr.GetPath("movies") - if err != nil { - t.Errorf("errog getting path for 'movies' dataset: %s", err.Error()) - return - } +// path, err := mr.GetPath("movies") +// if err != nil { +// t.Errorf("errog getting path for 'movies' dataset: %s", err.Error()) +// return +// } - // ns, err := mr.Namespace(30, 0) - // if err != nil { - // t.Errorf("error getting repo namespace: %s", err.Error()) - // return - // } - - // for _, n := range ns { - // fmt.Println(n) - // } - - qres := &repo.DatasetRef{} - if err = req.Run(&RunParams{ - Dataset: &dataset.Dataset{ - QueryString: "select * from movies", - }}, qres); err != nil { - t.Errorf("error running query: %s", err.Error()) - return - } +// // ns, err := mr.Namespace(30, 0) +// // if err != nil { +// // t.Errorf("error getting repo namespace: %s", err.Error()) +// // return +// // } + +// // for _, n := range ns { +// // fmt.Println(n) +// // } + +// qres := &repo.DatasetRef{} +// if err = req.Run(&RunParams{ +// Dataset: &dataset.Dataset{ +// QueryString: "select * from movies", +// }}, qres); err != nil { +// t.Errorf("error running query: %s", err.Error()) +// return +// } - cases := []struct { - p *DatasetQueriesParams - res []*repo.DatasetRef - err string - }{ - {&DatasetQueriesParams{}, []*repo.DatasetRef{}, "path is required"}, - {&DatasetQueriesParams{Path: path.String()}, []*repo.DatasetRef{&repo.DatasetRef{}}, ""}, - // TODO: add more tests - } +// cases := []struct { +// p *DatasetQueriesParams +// res []*repo.DatasetRef +// err string +// }{ +// {&DatasetQueriesParams{}, []*repo.DatasetRef{}, "path is required"}, +// {&DatasetQueriesParams{Path: path.String()}, []*repo.DatasetRef{&repo.DatasetRef{}}, ""}, +// // TODO: add more tests +// } - for i, c := range cases { - got := []*repo.DatasetRef{} - err := req.DatasetQueries(c.p, &got) - if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { - t.Errorf("case %d error mismatch: expected: %s, got: %s", i, c.err, err) - continue - } +// for i, c := range cases { +// got := []*repo.DatasetRef{} +// err := req.DatasetQueries(c.p, &got) +// if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { +// t.Errorf("case %d error mismatch: expected: %s, got: %s", i, c.err, err) +// continue +// } - fmt.Println(got) +// // fmt.Println(got) - if len(c.res) != len(got) { - t.Errorf("case %d returned wrong number of responses. exepected: %d, got %d", i, len(c.res), len(got)) - continue - } - } -} +// if len(c.res) != len(got) { +// t.Errorf("case %d returned wrong number of responses. exepected: %d, got %d", i, len(c.res), len(got)) +// continue +// } +// } +// } diff --git a/p2p/record.go b/p2p/record.go index 8b4963504..e4cf30830 100644 --- a/p2p/record.go +++ b/p2p/record.go @@ -8,7 +8,7 @@ import ( ) // TODO - work in progress -func (qn *QriNode) PutQueryKey(key datastore.Key, q *dataset.Query) error { +func (qn *QriNode) PutQueryKey(key datastore.Key, q *dataset.Transform) error { data, err := q.MarshalJSON() if err != nil { return err diff --git a/repo/dataset_ref.go b/repo/dataset_ref.go index bca9620a5..d34e4f89d 100644 --- a/repo/dataset_ref.go +++ b/repo/dataset_ref.go @@ -16,7 +16,7 @@ import ( // be stored as metadata within the dataset itself. type DatasetRef struct { // The dataset being referenced - Dataset *dataset.Dataset `json:"dataset"` + Dataset *dataset.Dataset `json:"dataset,omitempty"` // Unique name reference for this dataset Name string `json:"name,omitempty"` // Content-addressed path for this dataset diff --git a/repo/graph.go b/repo/graph.go index 03324131c..79f9dbc32 100644 --- a/repo/graph.go +++ b/repo/graph.go @@ -73,7 +73,7 @@ func QueriesMap(nodes map[string]*dsgraph.Node) (qs map[string]datastore.Key) { for path, node := range nodes { if node.Type == dsgraph.NtDataset && len(node.Links) > 0 { for _, l := range node.Links { - if l.To.Type == dsgraph.NtQuery { + if l.To.Type == dsgraph.NtTransform { qs[path] = datastore.NewKey(l.To.Path) } } @@ -87,7 +87,7 @@ func QueriesMap(nodes map[string]*dsgraph.Node) (qs map[string]datastore.Key) { func DatasetQueries(nodes map[string]*dsgraph.Node) (qs map[string]datastore.Key) { qs = map[string]datastore.Key{} for path, node := range nodes { - if node.Type == dsgraph.NtQuery && len(node.Links) > 0 { + if node.Type == dsgraph.NtTransform && len(node.Links) > 0 { for _, l := range node.Links { if l.To.Type == dsgraph.NtDataset { // qs[path] = datastore.NewKey(l.To.Path) @@ -131,7 +131,7 @@ func (nl NodeList) nodesFromDatasetRef(r Repo, ref *DatasetRef) *dsgraph.Node { root.AddLinks(dsgraph.Link{ From: root, - To: nl.node(dsgraph.NtData, ds.Data.String()), + To: nl.node(dsgraph.NtData, ds.Data), }) if ds.Previous.String() != "/" { @@ -140,35 +140,38 @@ func (nl NodeList) nodesFromDatasetRef(r Repo, ref *DatasetRef) *dsgraph.Node { To: nl.node(dsgraph.NtDataset, ds.Previous.String()), }) } - // if ds.Commit.Path().String() != "" { - // commit := &dsgraph.Node{Type: dsgraph.NtCommit, Path: ds.Commit.Path()} - // root.AddLinks(dsgraph.Link{From: root, To: data}) - // } + if ds.Commit != nil && ds.Commit.Path().String() != "" { + commit := &dsgraph.Node{Type: dsgraph.NtCommit, Path: ds.Commit.Path().String()} + root.AddLinks(dsgraph.Link{From: root, To: commit}) + } + if ds.AbstractStructure != nil && ds.AbstractStructure.Path().String() != "" { root.AddLinks(dsgraph.Link{ From: root, To: nl.node(dsgraph.NtAbstStructure, ds.AbstractStructure.Path().String()), }) } - if ds.Query != nil && ds.Query.Path().String() != "" { - if q, err := dsfs.LoadQuery(r.Store(), ds.Query.Path()); err == nil { - query := nl.node(dsgraph.NtQuery, ds.Query.Path().String()) - if q.Abstract != nil && q.Abstract.Path().String() != "" { - query.AddLinks(dsgraph.Link{ - From: query, - To: nl.node(dsgraph.NtAbstQuery, q.Abstract.Path().String()), - }) - } + + if ds.Transform != nil && ds.Transform.Path().String() != "" { + if q, err := dsfs.LoadTransform(r.Store(), ds.Transform.Path()); err == nil { + trans := nl.node(dsgraph.NtTransform, ds.Transform.Path().String()) for _, ref := range q.Resources { - query.AddLinks(dsgraph.Link{ - From: query, + trans.AddLinks(dsgraph.Link{ + From: trans, To: nl.node(dsgraph.NtDataset, ref.Path().String()), }) } - root.AddLinks(dsgraph.Link{From: root, To: query}) + root.AddLinks(dsgraph.Link{From: root, To: trans}) } } + if ds.AbstractTransform != nil && ds.AbstractTransform.Path().String() != "" { + root.AddLinks(dsgraph.Link{ + From: root, + To: nl.node(dsgraph.NtAbstTransform, ds.AbstractTransform.Path().String()), + }) + } + return root } diff --git a/repo/graph_test.go b/repo/graph_test.go index 5512d3fe6..405eae39c 100644 --- a/repo/graph_test.go +++ b/repo/graph_test.go @@ -23,7 +23,7 @@ func TestRepoGraph(t *testing.T) { return } - expect := 9 + expect := 8 count := 0 for range nodes { count++ @@ -98,11 +98,9 @@ func makeTestRepo() (Repo, error) { ds2 := &dataset.Dataset{ Title: "dataset 2", Previous: datastore.NewKey(""), - Query: &dataset.Query{ - Abstract: &dataset.AbstractQuery{ - Syntax: "sql", - Statement: "select * from a,b where b.id = 'foo'", - }, + Transform: &dataset.Transform{ + Syntax: "sql", + Data: "select * from a,b where b.id = 'foo'", Resources: map[string]*dataset.Dataset{ "a": dataset.NewDatasetRef(datastore.NewKey("/path/to/a")), "b": dataset.NewDatasetRef(datastore.NewKey("/path/to/b")), @@ -118,7 +116,7 @@ func makeTestRepo() (Repo, error) { } data1p, _ := store.Put(memfs.NewMemfileBytes("data1", []byte("dataset_1")), true) - ds1.Data = data1p + ds1.Data = data1p.String() ds1p, err := dsfs.SaveDataset(store, ds1, true) if err != nil { return nil, fmt.Errorf("error putting dataset: %s", err.Error()) @@ -127,7 +125,7 @@ func makeTestRepo() (Repo, error) { r.PutName("ds1", ds1p) data2p, _ := store.Put(memfs.NewMemfileBytes("data2", []byte("dataset_2")), true) - ds2.Data = data2p + ds2.Data = data2p.String() ds2p, err := dsfs.SaveDataset(store, ds2, true) if err != nil { return nil, fmt.Errorf("error putting dataset: %s", err.Error()) diff --git a/repo/test/test_repo.go b/repo/test/test_repo.go index 23b44577b..ffcc29074 100644 --- a/repo/test/test_repo.go +++ b/repo/test/test_repo.go @@ -53,7 +53,7 @@ func NewTestRepo() (mr repo.Repo, err error) { return } - ds.Data = datakey + ds.Data = datakey.String() dskey, err = dsfs.SaveDataset(ms, ds, true) if err != nil { From 92e42aeef050db0db71e166f170a74de68de03ea Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Fri, 8 Dec 2017 10:34:04 -0500 Subject: [PATCH 02/10] feat(RPC): change default port, provide RPC listener so we've been planning to do this for a while, but a proper reason to do so hadn't come up until we needed to be able to use the CLI while a server is running (while ssh'd into a container). This commit is a spike that only supports DatasetRequests for now while I work out the details of this pattern. also, closes #163 --- api/config.go | 15 ++++++++----- api/handlers/datasets.go | 2 +- api/server.go | 28 ++++++++++++++++++++++++ cmd/add.go | 4 ++-- cmd/export.go | 2 +- cmd/info.go | 2 +- cmd/init-ipfs.go | 10 ++++++++- cmd/remove.go | 2 +- cmd/rename.go | 2 +- cmd/repo.go | 31 ++++++++++++++++++++++++++ cmd/run.go | 10 +++++++-- cmd/server.go | 8 ++++--- cmd/update.go | 2 +- cmd/validate.go | 4 ++-- core/core.go | 34 +++++++++++++++++++++++++++++ core/datasets.go | 47 +++++++++++++++++++++++++++++++++++++++- core/datasets_test.go | 16 +++++++------- core/history.go | 2 ++ core/peers.go | 2 ++ core/profile.go | 2 ++ core/queries.go | 4 +++- core/search.go | 2 ++ 22 files changed, 200 insertions(+), 31 deletions(-) create mode 100644 core/core.go diff --git a/api/config.go b/api/config.go index 7bf5b6ec8..b66e07b2c 100644 --- a/api/config.go +++ b/api/config.go @@ -11,14 +11,17 @@ const ( DEVELOP_MODE = "develop" PRODUCTION_MODE = "production" TEST_MODE = "test" + DefaultPort = "2503" + DefaultRPCPort = "2504" ) func DefaultConfig() *Config { return &Config{ - Logger: logging.DefaultLogger, - Mode: "develop", - Port: "8080", - Online: true, + Logger: logging.DefaultLogger, + Mode: "develop", + Port: DefaultPort, + RPCPort: DefaultRPCPort, + Online: true, } } @@ -38,6 +41,8 @@ type Config struct { Mode string // port to listen on, will be read from PORT env variable if present. Port string + // port to listen for RPC calls on, if empty server will not register a RPC listener + RPCPort string // root url for service UrlRoot string // DNS service discovery. Should be either "env" or "dns", default is env @@ -67,7 +72,7 @@ type Config struct { func (cfg *Config) Validate() (err error) { // make sure port is set if cfg.Port == "" { - cfg.Port = "8080" + cfg.Port = DefaultPort } err = requireConfigStrings(map[string]string{ diff --git a/api/handlers/datasets.go b/api/handlers/datasets.go index 2e69019b5..859f7e035 100644 --- a/api/handlers/datasets.go +++ b/api/handlers/datasets.go @@ -17,7 +17,7 @@ import ( ) func NewDatasetHandlers(log logging.Logger, r repo.Repo) *DatasetHandlers { - req := core.NewDatasetRequests(r) + req := core.NewDatasetRequests(r, nil) h := DatasetHandlers{*req, log, r} return &h } diff --git a/api/server.go b/api/server.go index 0439c934d..cef80e4fb 100644 --- a/api/server.go +++ b/api/server.go @@ -2,10 +2,13 @@ package api import ( "fmt" + "net" "net/http" + "net/rpc" "github.com/datatogether/api/apiutil" "github.com/qri-io/qri/api/handlers" + "github.com/qri-io/qri/core" "github.com/qri-io/qri/logging" "github.com/qri-io/qri/p2p" "github.com/qri-io/qri/repo" @@ -71,10 +74,35 @@ func (s *Server) Serve() (err error) { server := &http.Server{} server.Handler = NewServerRoutes(s) s.log.Infof("starting api server on port %s", s.cfg.Port) + go s.ServeRPC() // http.ListenAndServe will not return unless there's an error return StartServer(s.cfg, server) } +// ServeRPC checks for a configured RPC port, and registers a listner if so +func (s *Server) ServeRPC() { + if s.cfg.RPCPort == "" { + return + } + + listener, err := net.Listen("tcp", fmt.Sprintf(":%s", s.cfg.RPCPort)) + if err != nil { + s.log.Infof("RPC listen on port %s error: %s", s.cfg.RPCPort, err) + return + } + + for _, rcvr := range core.Receivers(s.qriNode) { + if err := rpc.Register(rcvr); err != nil { + s.log.Infof("error registering RPC receiver %s: %s", rcvr.CoreRequestsName(), err.Error()) + return + } + } + + s.log.Infof("accepting RPC requests on port %s", s.cfg.RPCPort) + rpc.Accept(listener) + return +} + // NewServerRoutes returns a Muxer that has all API routes func NewServerRoutes(s *Server) *http.ServeMux { m := http.NewServeMux() diff --git a/cmd/add.go b/cmd/add.go index cf16eebb0..28d846a21 100644 --- a/cmd/add.go +++ b/cmd/add.go @@ -34,7 +34,7 @@ var datasetAddCmd = &cobra.Command{ ErrExit(fmt.Errorf("please provide a --name")) } - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) root := strings.TrimSuffix(args[0], "/"+dsfs.PackageFileDataset.String()) p := &core.AddParams{ @@ -69,7 +69,7 @@ func initDataset() { metaFile, err = loadFileIfPath(addDsMetaFilepath) ExitIfErr(err) - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) p := &core.InitDatasetParams{ Name: addDsName, diff --git a/cmd/export.go b/cmd/export.go index 5c7d4587d..45108ac98 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -30,7 +30,7 @@ var exportCmd = &cobra.Command{ } r := GetRepo(false) - req := core.NewDatasetRequests(r) + req := core.NewDatasetRequests(r, nil) p := &core.GetDatasetParams{ Name: args[0], diff --git a/cmd/info.go b/cmd/info.go index 1a35cf346..b1e0967b6 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -34,7 +34,7 @@ var infoCmd = &cobra.Command{ } } - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) for i, arg := range args { rt, ref := dsfs.RefType(arg) diff --git a/cmd/init-ipfs.go b/cmd/init-ipfs.go index 8ca6f3b2f..d3d96cce7 100644 --- a/cmd/init-ipfs.go +++ b/cmd/init-ipfs.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/ipfs/go-datastore" ipfs "github.com/qri-io/cafs/ipfs" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -10,13 +11,20 @@ var ( initIpfsConfigFile string ) +// defaultDatasets is a hard-coded dataset added when a new qri repo is created +// this hash must always be available +var defaultDatasets = []datastore.Key{ + // fivethirtyeight comic characters + datastore.NewKey("/ipfs/QmcqkHFA2LujZxY38dYZKmxsUstN4unk95azBjwEhwrnM6"), +} + // initCmd represents the init command var initIpfsCmd = &cobra.Command{ Use: "init-ipfs", Short: "Initialize an ipfs repository", Long: ``, Run: func(cmd *cobra.Command, args []string) { - err := ipfs.InitRepo(viper.GetString(IpfsFsPath), initIpfsConfigFile) + err := ipfs.InitRepo(viper.GetString(IpfsFsPath), initIpfsConfigFile, defaultDatasets) ExitIfErr(err) }, } diff --git a/cmd/remove.go b/cmd/remove.go index 4bf8b6e9e..2612aa7e6 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -19,7 +19,7 @@ var datasetRemoveCmd = &cobra.Command{ ErrExit(fmt.Errorf("please specify a dataset path or name to get the info of")) } - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) for _, arg := range args { rt, ref := dsfs.RefType(arg) diff --git a/cmd/rename.go b/cmd/rename.go index 1bcfa8f83..9a2a05646 100644 --- a/cmd/rename.go +++ b/cmd/rename.go @@ -18,7 +18,7 @@ var datasetRenameCmd = &cobra.Command{ ErrExit(fmt.Errorf("please provide current & new dataset names")) } - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) p := &core.RenameParams{ Current: args[0], New: args[1], diff --git a/cmd/repo.go b/cmd/repo.go index 5d9a5dae5..fc3df164b 100644 --- a/cmd/repo.go +++ b/cmd/repo.go @@ -1,6 +1,10 @@ package cmd import ( + "net" + "net/rpc" + "strings" + ipfs "github.com/qri-io/cafs/ipfs" "github.com/qri-io/qri/repo" "github.com/qri-io/qri/repo/fs" @@ -25,6 +29,33 @@ func GetRepo(online bool) repo.Repo { return r } +// RepoOrClient returns either a +func RepoOrClient(online bool) (repo.Repo, *rpc.Client) { + if fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { + cfg.FsRepoPath = viper.GetString(IpfsFsPath) + cfg.Online = online + }); err == nil { + id := "" + if fs.Node().PeerHost != nil { + id = fs.Node().PeerHost.ID().Pretty() + } + + r, err := fs_repo.NewRepo(fs, viper.GetString(QriRepoPath), id) + ExitIfErr(err) + return r, nil + } else if strings.Contains(err.Error(), "lock") { + // TODO - bad bad hardcode + conn, err := net.Dial("tcp", ":2504") + if err != nil { + ErrExit(err) + } + return nil, rpc.NewClient(conn) + } else { + ErrExit(err) + } + return nil, nil +} + func GetIpfsFilestore(online bool) *ipfs.Filestore { fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { cfg.FsRepoPath = viper.GetString(IpfsFsPath) diff --git a/cmd/run.go b/cmd/run.go index 09cdc11d1..1b14ef74f 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -59,7 +59,13 @@ var runCmd = &cobra.Command{ results, err := ioutil.ReadAll(f) ExitIfErr(err) - PrintResults(res.Dataset.Structure, results, res.Dataset.Structure.Format) + switch cmd.Flag("format").Value.String() { + case "csv", "json": + fmt.Printf("%s", string(results)) + default: + PrintResults(res.Dataset.Structure, results, res.Dataset.Structure.Format) + } + }, } @@ -67,6 +73,6 @@ func init() { RootCmd.AddCommand(runCmd) // runCmd.Flags().StringP("save", "s", "", "save the resulting dataset to a given address") runCmd.Flags().StringP("output", "o", "", "file to write to") - runCmd.Flags().StringP("format", "f", "csv", "set output format [csv,json]") + runCmd.Flags().StringP("format", "f", "", "set output format [csv,json]") runCmd.Flags().StringVarP(&runCmdName, "name", "n", "", "save output to name") } diff --git a/cmd/server.go b/cmd/server.go index 6ca60fe05..4cc756948 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -67,9 +67,9 @@ var serverCmd = &cobra.Command{ } func init() { - serverCmd.Flags().StringVarP(&serverCmdPort, "port", "p", "3000", "port to start server on") + serverCmd.Flags().StringVarP(&serverCmdPort, "port", "p", api.DefaultPort, "port to start server on") serverCmd.Flags().BoolVarP(&serverInitIpfs, "init-ipfs", "", false, "initialize a new default ipfs repo if empty") - serverCmd.Flags().BoolVarP(&serverMemOnly, "mem-only", "", false, "run qri entirely in-memory") + serverCmd.Flags().BoolVarP(&serverMemOnly, "mem-only", "", false, "run qri entirely in-memory, persisting nothing") serverCmd.Flags().BoolVarP(&serverOffline, "offline", "", false, "disable networking") RootCmd.AddCommand(serverCmd) } @@ -80,7 +80,9 @@ func initRepoIfEmpty(repoPath, configPath string) error { if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { return err } - return ipfs.InitRepo(repoPath, configPath) + if err := ipfs.InitRepo(repoPath, configPath, defaultDatasets); err != nil { + return err + } } } return nil diff --git a/cmd/update.go b/cmd/update.go index cad143e15..1c823807e 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -46,7 +46,7 @@ var updateCmd = &cobra.Command{ ErrExit(fmt.Errorf("either a metadata or data option is required")) } - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) p := &core.GetDatasetParams{ Name: args[0], diff --git a/cmd/validate.go b/cmd/validate.go index 46eb633a5..b9dcd9bb3 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -29,7 +29,7 @@ and check each of it's rows against the constraints listed in the dataset's fields.`, Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) for _, arg := range args { rt, ref := dsfs.RefType(arg) p := &core.ValidateDatasetParams{} @@ -78,7 +78,7 @@ func validateDataset() { metaFile, err = loadFileIfPath(validateDsMetaFilepath) ExitIfErr(err) - req := core.NewDatasetRequests(GetRepo(false)) + req := core.NewDatasetRequests(RepoOrClient(false)) p := &core.ValidateDatasetParams{ Name: validateDsName, diff --git a/core/core.go b/core/core.go new file mode 100644 index 000000000..8702feecb --- /dev/null +++ b/core/core.go @@ -0,0 +1,34 @@ +package core + +import ( + "github.com/qri-io/qri/p2p" +) + +// CoreRequests defines a set of core methods +type CoreRequests interface { + // CoreRequestsName confirms participation in the CoreRequests interface while + // also giving a human readable string for logging purposes + CoreRequestsName() string +} + +// Requests returns a slice of CoreRequests that defines the full local +// API of core methods +func Receivers(node *p2p.QriNode) []CoreRequests { + r := node.Repo + return []CoreRequests{ + NewDatasetRequests(r, nil), + NewHistoryRequests(r), + NewPeerRequests(r, node), + NewProfileRequests(r), + NewQueryRequests(r), + NewSearchRequests(r), + } +} + +// func RemoteClient(addr string) (*rpc.Client, error) { +// conn, err := net.Dial("tcp", addr) +// if err != nil { +// return nil, fmt.Errorf("dial error: %s", err) +// } +// return rpc.NewClient(conn), nil +// } diff --git a/core/datasets.go b/core/datasets.go index 114370ba0..8933d474c 100644 --- a/core/datasets.go +++ b/core/datasets.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "net/rpc" "path/filepath" "strings" "time" @@ -25,15 +26,27 @@ import ( type DatasetRequests struct { repo repo.Repo + cli *rpc.Client } -func NewDatasetRequests(r repo.Repo) *DatasetRequests { +func (d DatasetRequests) CoreRequestsName() string { return "datasets" } + +func NewDatasetRequests(r repo.Repo, cli *rpc.Client) *DatasetRequests { + if r != nil && cli != nil { + panic(fmt.Errorf("both repo and client supplied to NewDatasetRequests")) + } + return &DatasetRequests{ repo: r, + cli: cli, } } func (d *DatasetRequests) List(p *ListParams, res *[]*repo.DatasetRef) error { + if d.cli != nil { + return d.cli.Call("DatasetRequests.List", p, res) + } + store := d.repo.Store() // ensure valid limit value if p.Limit <= 0 { @@ -76,6 +89,10 @@ type GetDatasetParams struct { } func (d *DatasetRequests) Get(p *GetDatasetParams, res *repo.DatasetRef) error { + if d.cli != nil { + return d.cli.Call("DatasetRequests.Get", p, res) + } + store := d.repo.Store() ds, err := dsfs.LoadDataset(store, p.Path) if err != nil { @@ -109,6 +126,10 @@ type InitDatasetParams struct { // InitDataset creates a new qri dataset from a source of data func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef) error { + if r.cli != nil { + return r.cli.Call("DatasetRequests.InitDataset", p, res) + } + var ( rdr io.Reader store = r.repo.Store() @@ -245,6 +266,10 @@ type UpdateParams struct { // Update adds a history entry, updating a dataset func (r *DatasetRequests) Update(p *UpdateParams, res *repo.DatasetRef) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.Update", p, res) + } + var ( name string prevpath datastore.Key @@ -331,6 +356,10 @@ type RenameParams struct { } func (r *DatasetRequests) Rename(p *RenameParams, res *repo.DatasetRef) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.Rename", p, res) + } + if p.Current == "" { return fmt.Errorf("current name is required to rename a dataset") } @@ -373,6 +402,10 @@ type DeleteParams struct { } func (r *DatasetRequests) Delete(p *DeleteParams, ok *bool) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.List", p, ok) + } + if p.Name == "" && p.Path.String() == "" { return fmt.Errorf("either name or path is required") } @@ -418,6 +451,10 @@ type StructuredData struct { } func (r *DatasetRequests) StructuredData(p *StructuredDataParams, data *StructuredData) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.StructuredData", p, data) + } + var ( file cafs.File d []byte @@ -480,6 +517,10 @@ type AddParams struct { } func (r *DatasetRequests) AddDataset(p *AddParams, res *repo.DatasetRef) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.AddDataset", p, res) + } + fs, ok := r.repo.Store().(*ipfs.Filestore) if !ok { return fmt.Errorf("can only add datasets when running an IPFS filestore") @@ -526,6 +567,10 @@ type ValidateDatasetParams struct { } func (r *DatasetRequests) Validate(p *ValidateDatasetParams, errors *dataset.Dataset) (err error) { + if r.cli != nil { + return r.cli.Call("DatasetRequests.Validate", p, errors) + } + // store := Store(cmd, args) // errs, err := history.Validate(store) // ExitIfErr(err) diff --git a/core/datasets_test.go b/core/datasets_test.go index 821d3c445..3c21957c2 100644 --- a/core/datasets_test.go +++ b/core/datasets_test.go @@ -49,7 +49,7 @@ func TestDatasetRequestsInit(t *testing.T) { return } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := &repo.DatasetRef{} err := req.InitDataset(c.p, got) @@ -103,7 +103,7 @@ func TestDatasetRequestsList(t *testing.T) { // TODO: re-enable {&ListParams{OrderBy: "name", Limit: 30, Offset: 0}, []*repo.DatasetRef{cities, counter, movies}, ""}, } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := []*repo.DatasetRef{} err := req.List(c.p, &got) @@ -156,7 +156,7 @@ func TestDatasetRequestsGet(t *testing.T) { {&GetDatasetParams{Path: path, Name: "cats", Hash: "123"}, moviesDs, ""}, } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := &repo.DatasetRef{} err := req.Get(c.p, got) @@ -198,7 +198,7 @@ func TestDatasetRequestsUpdate(t *testing.T) { // {&UpdateParams{Path: path, Name: "cats", Hash: "123"}, moviesDs, ""}, } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := &repo.DatasetRef{} err := req.Update(c.p, got) @@ -230,7 +230,7 @@ func TestDatasetRequestsRename(t *testing.T) { {&RenameParams{Current: "new_movies", New: "new_movies"}, "", "name 'new_movies' already exists"}, } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := &repo.DatasetRef{} err := req.Rename(c.p, got) @@ -269,7 +269,7 @@ func TestDatasetRequestsDelete(t *testing.T) { {&DeleteParams{Path: path}, nil, ""}, } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := false err := req.Delete(c.p, &got) @@ -313,7 +313,7 @@ func TestDatasetRequestsStructuredData(t *testing.T) { {&StructuredDataParams{Format: dataset.JSONDataFormat, Path: archivePath, Limit: 0, Offset: 0, All: true}, 0, ""}, } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := &StructuredData{} err := req.StructuredData(c.p, got) @@ -361,7 +361,7 @@ func TestDatasetRequestsAddDataset(t *testing.T) { return } - req := NewDatasetRequests(mr) + req := NewDatasetRequests(mr, nil) for i, c := range cases { got := &repo.DatasetRef{} err := req.AddDataset(c.p, got) diff --git a/core/history.go b/core/history.go index 5f760b401..2d5eef335 100644 --- a/core/history.go +++ b/core/history.go @@ -12,6 +12,8 @@ type HistoryRequests struct { repo repo.Repo } +func (d HistoryRequests) CoreRequestsName() string { return "history" } + func NewHistoryRequests(r repo.Repo) *HistoryRequests { return &HistoryRequests{ repo: r, diff --git a/core/peers.go b/core/peers.go index 9562f0726..0f8ccad62 100644 --- a/core/peers.go +++ b/core/peers.go @@ -24,6 +24,8 @@ type PeerRequests struct { qriNode *p2p.QriNode } +func (d PeerRequests) CoreRequestsName() string { return "peers" } + func (d *PeerRequests) List(p *ListParams, res *[]*profile.Profile) error { replies := make([]*profile.Profile, p.Limit) i := 0 diff --git a/core/profile.go b/core/profile.go index 3f6eae004..92dc58aca 100644 --- a/core/profile.go +++ b/core/profile.go @@ -15,6 +15,8 @@ type ProfileRequests struct { repo repo.Repo } +func (d ProfileRequests) CoreRequestsName() string { return "profile" } + func NewProfileRequests(r repo.Repo) *ProfileRequests { return &ProfileRequests{ repo: r, diff --git a/core/queries.go b/core/queries.go index f350ff390..f99359413 100644 --- a/core/queries.go +++ b/core/queries.go @@ -16,6 +16,8 @@ type QueryRequests struct { repo repo.Repo } +func (d QueryRequests) CoreRequestsName() string { return "queries" } + func NewQueryRequests(r repo.Repo) *QueryRequests { return &QueryRequests{ repo: r, @@ -215,7 +217,7 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { if err := dsfs.DerefDatasetTransform(store, ds); err != nil { return fmt.Errorf("error dereferencing dataset query: %s", err.Error()) } - fmt.Println(ds.AbstractTransform.Path().String()) + // fmt.Println(ds.AbstractTransform.Path().String()) ref := &repo.DatasetRef{Name: p.SaveName, Path: dspath, Dataset: ds} diff --git a/core/search.go b/core/search.go index 77e097be9..3954a329e 100644 --- a/core/search.go +++ b/core/search.go @@ -14,6 +14,8 @@ type SearchRequests struct { // node *p2p.QriNode } +func (d SearchRequests) CoreRequestsName() string { return "search" } + func NewSearchRequests(r repo.Repo) *SearchRequests { return &SearchRequests{ repo: r, From 05a9e2f321619bf355e1c0001769605f083bfc37 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Fri, 8 Dec 2017 11:46:40 -0500 Subject: [PATCH 03/10] feat(DefaultDatasets): first cuts on requesting default datasets when no config file is present on any CLI command, attempt to spin up a repo & grap default hashes. This'll definitly break stuff but we got a demo today so whatever closes #161 --- cmd/add.go | 8 +++++--- cmd/config.go | 13 ++++++++----- cmd/info.go | 3 ++- cmd/init-ipfs.go | 10 +--------- cmd/remove.go | 3 ++- cmd/rename.go | 5 +++-- cmd/repo.go | 41 ++++++++++++++++++++++++++--------------- cmd/root.go | 44 +++++++++++++++++++++++++++++++++++++++++++- cmd/server.go | 2 +- cmd/update.go | 3 ++- cmd/validate.go | 7 +++++-- p2p/bootstrap.go | 15 ++++++++++++++- 12 files changed, 112 insertions(+), 42 deletions(-) diff --git a/cmd/add.go b/cmd/add.go index 28d846a21..bac7273ed 100644 --- a/cmd/add.go +++ b/cmd/add.go @@ -34,7 +34,8 @@ var datasetAddCmd = &cobra.Command{ ErrExit(fmt.Errorf("please provide a --name")) } - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) root := strings.TrimSuffix(args[0], "/"+dsfs.PackageFileDataset.String()) p := &core.AddParams{ @@ -42,7 +43,7 @@ var datasetAddCmd = &cobra.Command{ Hash: root, } res := &repo.DatasetRef{} - err := req.AddDataset(p, res) + err = req.AddDataset(p, res) ExitIfErr(err) PrintInfo("Successfully added dataset %s: %s", addDsName, res.Path.String()) @@ -69,7 +70,8 @@ func initDataset() { metaFile, err = loadFileIfPath(addDsMetaFilepath) ExitIfErr(err) - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) p := &core.InitDatasetParams{ Name: addDsName, diff --git a/cmd/config.go b/cmd/config.go index d368dcd03..42cdc0c4e 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -51,7 +51,8 @@ func init() { } // initConfig reads in config file and ENV variables if set. -func initConfig() { +func initConfig() (created bool) { + var err error home := userHomeDir() SetNoColor() @@ -84,11 +85,13 @@ func initConfig() { viper.SetConfigName("config") // name of config file (without extension) viper.AddConfigPath(qriPath) // add QRI_PATH env var - err := EnsureConfigFile() + created, err = EnsureConfigFile() ExitIfErr(err) err = viper.ReadInConfig() ExitIfErr(err) + + return } func configFilepath() string { @@ -99,12 +102,12 @@ func configFilepath() string { return path } -func EnsureConfigFile() error { +func EnsureConfigFile() (bool, error) { if _, err := os.Stat(configFilepath()); os.IsNotExist(err) { fmt.Println("writing config file") - return WriteConfigFile(defaultCfg) + return true, WriteConfigFile(defaultCfg) } - return nil + return false, nil } func WriteConfigFile(cfg *Config) error { diff --git a/cmd/info.go b/cmd/info.go index b1e0967b6..4e821cec3 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -34,7 +34,8 @@ var infoCmd = &cobra.Command{ } } - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) for i, arg := range args { rt, ref := dsfs.RefType(arg) diff --git a/cmd/init-ipfs.go b/cmd/init-ipfs.go index d3d96cce7..8ca6f3b2f 100644 --- a/cmd/init-ipfs.go +++ b/cmd/init-ipfs.go @@ -1,7 +1,6 @@ package cmd import ( - "github.com/ipfs/go-datastore" ipfs "github.com/qri-io/cafs/ipfs" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -11,20 +10,13 @@ var ( initIpfsConfigFile string ) -// defaultDatasets is a hard-coded dataset added when a new qri repo is created -// this hash must always be available -var defaultDatasets = []datastore.Key{ - // fivethirtyeight comic characters - datastore.NewKey("/ipfs/QmcqkHFA2LujZxY38dYZKmxsUstN4unk95azBjwEhwrnM6"), -} - // initCmd represents the init command var initIpfsCmd = &cobra.Command{ Use: "init-ipfs", Short: "Initialize an ipfs repository", Long: ``, Run: func(cmd *cobra.Command, args []string) { - err := ipfs.InitRepo(viper.GetString(IpfsFsPath), initIpfsConfigFile, defaultDatasets) + err := ipfs.InitRepo(viper.GetString(IpfsFsPath), initIpfsConfigFile) ExitIfErr(err) }, } diff --git a/cmd/remove.go b/cmd/remove.go index 2612aa7e6..e1b5544c6 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -19,7 +19,8 @@ var datasetRemoveCmd = &cobra.Command{ ErrExit(fmt.Errorf("please specify a dataset path or name to get the info of")) } - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) for _, arg := range args { rt, ref := dsfs.RefType(arg) diff --git a/cmd/rename.go b/cmd/rename.go index 9a2a05646..66969fe11 100644 --- a/cmd/rename.go +++ b/cmd/rename.go @@ -18,13 +18,14 @@ var datasetRenameCmd = &cobra.Command{ ErrExit(fmt.Errorf("please provide current & new dataset names")) } - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) p := &core.RenameParams{ Current: args[0], New: args[1], } res := &repo.DatasetRef{} - err := req.Rename(p, res) + err = req.Rename(p, res) ExitIfErr(err) PrintSuccess("renamed dataset %s", res.Name) diff --git a/cmd/repo.go b/cmd/repo.go index fc3df164b..513287cff 100644 --- a/cmd/repo.go +++ b/cmd/repo.go @@ -1,11 +1,13 @@ package cmd import ( + "fmt" "net" "net/rpc" "strings" ipfs "github.com/qri-io/cafs/ipfs" + "github.com/qri-io/qri/core" "github.com/qri-io/qri/repo" "github.com/qri-io/qri/repo/fs" "github.com/spf13/viper" @@ -29,8 +31,25 @@ func GetRepo(online bool) repo.Repo { return r } +func GetIpfsFilestore(online bool) *ipfs.Filestore { + fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { + cfg.FsRepoPath = viper.GetString(IpfsFsPath) + cfg.Online = online + }) + ExitIfErr(err) + return fs +} + +func DatasetRequests(online bool) (*core.DatasetRequests, error) { + r, cli, err := RepoOrClient(online) + if err != nil { + return nil, err + } + return core.NewDatasetRequests(r, cli), nil +} + // RepoOrClient returns either a -func RepoOrClient(online bool) (repo.Repo, *rpc.Client) { +func RepoOrClient(online bool) (repo.Repo, *rpc.Client, error) { if fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { cfg.FsRepoPath = viper.GetString(IpfsFsPath) cfg.Online = online @@ -41,26 +60,18 @@ func RepoOrClient(online bool) (repo.Repo, *rpc.Client) { } r, err := fs_repo.NewRepo(fs, viper.GetString(QriRepoPath), id) - ExitIfErr(err) - return r, nil + return r, nil, err + } else if strings.Contains(err.Error(), "lock") { // TODO - bad bad hardcode conn, err := net.Dial("tcp", ":2504") if err != nil { - ErrExit(err) + return nil, nil, err } - return nil, rpc.NewClient(conn) + return nil, rpc.NewClient(conn), nil } else { - ErrExit(err) + return nil, nil, err } - return nil, nil -} -func GetIpfsFilestore(online bool) *ipfs.Filestore { - fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { - cfg.FsRepoPath = viper.GetString(IpfsFsPath) - cfg.Online = online - }) - ExitIfErr(err) - return fs + return nil, nil, fmt.Errorf("badbadnotgood") } diff --git a/cmd/root.go b/cmd/root.go index 9150ef5c8..b436c296a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,10 @@ package cmd import ( "flag" + "fmt" + "github.com/ipfs/go-datastore" + "github.com/qri-io/qri/core" + "github.com/qri-io/qri/repo" "os" "github.com/spf13/cobra" @@ -42,8 +46,46 @@ func Execute() { func init() { flag.Parse() - cobra.OnInitialize(initConfig) + cobra.OnInitialize(initialize) RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $QRI_PATH/config.json)") RootCmd.PersistentFlags().BoolVarP(&noColor, "no-color", "c", false, "disable colorized output") } + +func initialize() { + created := initConfig() + if created { + go addDefaultDatasets() + } +} + +// defaultDatasets is a hard-coded dataset added when a new qri repo is created +// these hashes should always/highly available +var defaultDatasets = map[string]datastore.Key{ + // fivethirtyeight comic characters + "comic_characters": datastore.NewKey("/ipfs/QmcqkHFA2LujZxY38dYZKmxsUstN4unk95azBjwEhwrnM6/dataset.json"), +} + +// Init sets up a repository with sensible defaults +func addDefaultDatasets() error { + req, err := DatasetRequests(true) + if err != nil { + return err + } + + for name, ds := range defaultDatasets { + fmt.Printf("attempting to add default dataset: %s\n", ds.String()) + res := &repo.DatasetRef{} + err := req.AddDataset(&core.AddParams{ + Hash: ds.String(), + Name: name, + }, res) + if err != nil { + fmt.Printf("add dataset %s error: %s\n", ds.String(), err.Error()) + return err + } + fmt.Printf("added default dataset: %s\n", ds.String()) + } + + return nil +} diff --git a/cmd/server.go b/cmd/server.go index 4cc756948..fbe6a4f07 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -80,7 +80,7 @@ func initRepoIfEmpty(repoPath, configPath string) error { if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { return err } - if err := ipfs.InitRepo(repoPath, configPath, defaultDatasets); err != nil { + if err := ipfs.InitRepo(repoPath, configPath); err != nil { return err } } diff --git a/cmd/update.go b/cmd/update.go index 1c823807e..3ec2b840c 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -46,7 +46,8 @@ var updateCmd = &cobra.Command{ ErrExit(fmt.Errorf("either a metadata or data option is required")) } - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) p := &core.GetDatasetParams{ Name: args[0], diff --git a/cmd/validate.go b/cmd/validate.go index b9dcd9bb3..7a719035f 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -29,7 +29,9 @@ and check each of it's rows against the constraints listed in the dataset's fields.`, Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) + for _, arg := range args { rt, ref := dsfs.RefType(arg) p := &core.ValidateDatasetParams{} @@ -78,7 +80,8 @@ func validateDataset() { metaFile, err = loadFileIfPath(validateDsMetaFilepath) ExitIfErr(err) - req := core.NewDatasetRequests(RepoOrClient(false)) + req, err := DatasetRequests(false) + ExitIfErr(err) p := &core.ValidateDatasetParams{ Name: validateDsName, diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index 6eac7eab4..ddfa01b8e 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -18,7 +18,20 @@ import ( // One day it would be super nice to bootstrap from a stored history & only // use these for first-round bootstrapping. var DefaultBootstrapAddresses = []string{ - "/ip4/35.192.124.143/tcp/4001/ipfs/QmQffqhgce94UFS9mSvvqcAWXQNr1bcZRM659VFakySair", + // "/ip4/35.192.124.143/tcp/4001/ipfs/QmQffqhgce94UFS9mSvvqcAWXQNr1bcZRM659VFakySair", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", + "/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", + "/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", + "/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", + "/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", + "/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", + "/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", + "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", } // Bootstrap samples a subset of peers & requests their peers list From 314f088bb90dc8329e19e26437da1e6013f7cb36 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Fri, 8 Dec 2017 15:24:48 -0500 Subject: [PATCH 04/10] feat(cmd.Search): added format to search, fix dockerfile --- Dockerfile | 8 ++++---- cmd/search.go | 19 +++++++++++++++++-- core/search.go | 4 ++-- p2p/bootstrap.go | 18 ++++-------------- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2da4eceed..257fcab52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,10 @@ LABEL maintainer="sparkle_pony_2000@qri.io" ADD . /go/src/github.com/qri-io/qri RUN cd /go/src/github.com/qri-io/qri -# RUN go get -u github.com/whyrusleeping/gx -# RUN go get -u github.com/whyrusleeping/gx-go -# RUN gx install -# RUN go get ./... +RUN go get -v github.com/briandowns/spinner github.com/datatogether/api/apiutil github.com/fatih/color github.com/ipfs/go-datastore github.com/olekukonko/tablewriter github.com/qri-io/analytics github.com/qri-io/bleve github.com/qri-io/dataset github.com/qri-io/dataset_sql github.com/qri-io/doggos github.com/sirupsen/logrus github.com/spf13/cobra github.com/spf13/viper + +RUN go get -u github.com/whyrusleeping/gx github.com/whyrusleeping/gx-go +RUN cd /go/src/github.com/qri-io/qri && pwd && gx install RUN go install github.com/qri-io/qri # set default port to 8080, default log level, QRI_PATH env, IPFS_PATH env diff --git a/cmd/search.go b/cmd/search.go index 4cc238758..760a98ed0 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -1,8 +1,10 @@ package cmd import ( + "encoding/json" "fmt" + "github.com/qri-io/dataset" "github.com/qri-io/qri/core" "github.com/qri-io/qri/repo" "github.com/spf13/cobra" @@ -45,13 +47,26 @@ var searchCmd = &cobra.Command{ err := req.Search(p, &res) ExitIfErr(err) - for i, ref := range res { - PrintDatasetRefInfo(i, ref) + outformat := cmd.Flag("format").Value.String() + + switch outformat { + case "": + for i, ref := range res { + PrintDatasetRefInfo(i, ref) + } + case dataset.JSONDataFormat.String(): + data, err := json.MarshalIndent(res, "", " ") + ExitIfErr(err) + fmt.Printf("%s\n", string(data)) + default: + ErrExit(fmt.Errorf("unrecognized format: %s", outformat)) } + }, } func init() { searchCmd.Flags().BoolVarP(&searchCmdReindex, "reindex", "r", false, "re-generate search index from scratch. might take a while.") + searchCmd.Flags().StringP("format", "f", "", "set output format [json]") RootCmd.AddCommand(searchCmd) } diff --git a/core/search.go b/core/search.go index 3954a329e..d35141984 100644 --- a/core/search.go +++ b/core/search.go @@ -30,8 +30,8 @@ func (d *SearchRequests) Search(p *repo.SearchParams, res *[]*repo.DatasetRef) e // return err // } - if s, ok := d.repo.(repo.Searchable); ok { - results, err := s.Search(*p) + if searchable, ok := d.repo.(repo.Searchable); ok { + results, err := searchable.Search(*p) if err != nil { return fmt.Errorf("error searching: %s", err.Error()) } diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index ddfa01b8e..3a036be5c 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -18,20 +18,7 @@ import ( // One day it would be super nice to bootstrap from a stored history & only // use these for first-round bootstrapping. var DefaultBootstrapAddresses = []string{ - // "/ip4/35.192.124.143/tcp/4001/ipfs/QmQffqhgce94UFS9mSvvqcAWXQNr1bcZRM659VFakySair", - "/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - "/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", - "/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", - "/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", - "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", - "/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", - "/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", - "/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", - "/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", - "/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", - "/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", - "/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", - "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", + "/ip4/35.192.124.143/tcp/4001/ipfs/QmXNqD5ATi1ejL4HNzUzDyeWn46hHgJTqA26JmYiUWERcb", } // Bootstrap samples a subset of peers & requests their peers list @@ -50,6 +37,9 @@ func (n *QriNode) Bootstrap(boostrapAddrs []string) { go func() { if err := n.Host.Connect(context.Background(), pi); err == nil { n.log.Infof("boostrapping to: %s", pi.ID.Pretty()) + if err = n.AddQriPeer(pi); err != nil { + n.log.Infof("error adding peer: %s", err.Error()) + } n.RequestPeersList(pi.ID) } else { n.log.Infof("error connecting to host: %s", err.Error()) From 351c8dd3e74781899ea52eb2c39aea4190870682 Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 11 Dec 2017 15:48:35 -0500 Subject: [PATCH 05/10] fix(p2p.Bootstrap): fixes to bootstrapping --- cmd/cmd.go | 22 ++++++++++++++-------- cmd/config.go | 2 +- p2p/bootstrap.go | 16 ++++++++-------- p2p/peers.go | 12 ++++++------ package.json | 4 ++-- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index bc2674608..9c7876ebf 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -4,8 +4,8 @@ import ( "fmt" "os" "path/filepath" - "runtime" + "github.com/mitchellh/go-homedir" "github.com/spf13/viper" ) @@ -41,14 +41,20 @@ func cachePath() string { } func userHomeDir() string { - if runtime.GOOS == "windows" { - home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") - if home == "" { - home = os.Getenv("USERPROFILE") - } - return home + dir, err := homedir.Dir() + if err != nil { + panic(err) } - return os.Getenv("HOME") + return dir + + // if runtime.GOOS == "windows" { + // home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + // if home == "" { + // home = os.Getenv("USERPROFILE") + // } + // return home + // } + // return os.Getenv("HOME") } func loadFileIfPath(path string) (file *os.File, err error) { diff --git a/cmd/config.go b/cmd/config.go index 42cdc0c4e..9aeda0bd5 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -77,7 +77,7 @@ func initConfig() (created bool) { ipfsFsPath := os.Getenv("IPFS_PATH") if ipfsFsPath == "" { - ipfsFsPath = "$HOME/.ipfs" + ipfsFsPath = filepath.Join(home, ".ipfs") } ipfsFsPath = strings.Replace(ipfsFsPath, "~", home, 1) viper.SetDefault(IpfsFsPath, ipfsFsPath) diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index 3a036be5c..52f686616 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -14,7 +14,7 @@ import ( // This boostrapping is specific to finding qri peers, which are IPFS peers that also // support the qri protocol. // (we also perform standard IPFS boostrapping when IPFS networking is enabled, and it's almost always enabled). -// These are addresses to public, qri nodes hosted by qri. +// These are addresses to public qri nodes hosted by qri, inc. // One day it would be super nice to bootstrap from a stored history & only // use these for first-round bootstrapping. var DefaultBootstrapAddresses = []string{ @@ -33,18 +33,18 @@ func (n *QriNode) Bootstrap(boostrapAddrs []string) { pinfos := toPeerInfos(peers) - for _, pi := range randomSubsetOfPeers(pinfos, 4) { - go func() { - if err := n.Host.Connect(context.Background(), pi); err == nil { - n.log.Infof("boostrapping to: %s", pi.ID.Pretty()) - if err = n.AddQriPeer(pi); err != nil { + for _, p := range randomSubsetOfPeers(pinfos, 4) { + go func(p pstore.PeerInfo) { + n.Host.Peerstore().AddAddrs(p.ID, p.Addrs, pstore.RecentlyConnectedAddrTTL) + if err := n.Host.Connect(context.Background(), p); err == nil { + n.log.Infof("boostrapping to: %s", p.ID.Pretty()) + if err = n.AddQriPeer(p); err != nil { n.log.Infof("error adding peer: %s", err.Error()) } - n.RequestPeersList(pi.ID) } else { n.log.Infof("error connecting to host: %s", err.Error()) } - }() + }(p) } } diff --git a/p2p/peers.go b/p2p/peers.go index 13a3be23a..808fcbb72 100644 --- a/p2p/peers.go +++ b/p2p/peers.go @@ -3,7 +3,7 @@ package p2p import ( "context" "fmt" - "time" + // "time" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" @@ -13,10 +13,10 @@ func (n *QriNode) AddQriPeer(pinfo pstore.PeerInfo) error { // add this peer to our store n.QriPeers.AddAddrs(pinfo.ID, pinfo.Addrs, pstore.TempAddrTTL) - if profile, _ := n.Repo.Peers().GetPeer(pinfo.ID); profile != nil { - // we've already seen this peer - return nil - } + // if profile, _ := n.Repo.Peers().GetPeer(pinfo.ID); profile != nil { + // // we've already seen this peer + // return nil + // } if err := n.RequestProfileInfo(pinfo); err != nil { return err @@ -24,7 +24,7 @@ func (n *QriNode) AddQriPeer(pinfo pstore.PeerInfo) error { // some time later ask for a list of their peers, you know, "for a friend" go func() { - time.Sleep(time.Second * 2) + // time.Sleep(time.Second * 2) n.RequestPeersList(pinfo.ID) }() diff --git a/package.json b/package.json index 0c6677c19..7d85bfdcf 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,9 @@ }, "gxDependencies": [ { - "hash": "QmdKL1GVaUaDVt3JUWiYQSLYRsJMym2KRWxsiXAeEU6pzX", + "hash": "QmViBzgruNUoLNBnXcx8YWbDNwV8MNGEGKkLo6JGetygdw", "name": "go-ipfs", - "version": "0.4.12" + "version": "0.4.13" }, { "author": "whyrusleeping", From d4778fe8aa66bf9690e597ae678309fe69371747 Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 11 Dec 2017 16:54:31 -0500 Subject: [PATCH 06/10] feat(api.ServeRPC): serve core methods over RPC this commit builds upon intial work to make all core methods available as RPC. this isn't finished as of yet, given that all in and out params will need to conform to encoding/gob specs, which basically boils down to using only Exported types. I think this is a good opportunity to have *all* params to core methods be declared in the core package itself (these could be aliases to underlying types if needed). TODO: * finish implementation * write test suite that works core methods over RPC --- api/handlers/history.go | 2 +- api/handlers/peers.go | 2 +- api/handlers/profile.go | 8 ++-- api/handlers/queries.go | 2 +- api/handlers/search.go | 2 +- cmd/profile.go | 74 +++++++++++++++++++++++++++++++ cmd/queries.go | 5 ++- cmd/repo.go | 24 ++++++++++ cmd/run.go | 4 +- cmd/search.go | 5 ++- core/core.go | 10 ++--- core/history.go | 12 ++++- core/peers.go | 48 +++++++++++++++----- core/profile.go | 98 ++++++++++++++++++++++++++++++++++++++--- core/queries.go | 27 +++++++++++- core/search.go | 16 ++++++- 16 files changed, 299 insertions(+), 40 deletions(-) create mode 100644 cmd/profile.go diff --git a/api/handlers/history.go b/api/handlers/history.go index c34e5d502..e8891b0a2 100644 --- a/api/handlers/history.go +++ b/api/handlers/history.go @@ -17,7 +17,7 @@ type HistoryHandlers struct { } func NewHistoryHandlers(log logging.Logger, r repo.Repo) *HistoryHandlers { - req := core.NewHistoryRequests(r) + req := core.NewHistoryRequests(r, nil) h := HistoryHandlers{*req, log} return &h } diff --git a/api/handlers/peers.go b/api/handlers/peers.go index 0bb353872..2c60cc89f 100644 --- a/api/handlers/peers.go +++ b/api/handlers/peers.go @@ -14,7 +14,7 @@ import ( ) func NewPeerHandlers(log logging.Logger, r repo.Repo, node *p2p.QriNode) *PeerHandlers { - req := core.NewPeerRequests(r, node) + req := core.NewPeerRequests(node, nil) h := PeerHandlers{*req, log} return &h } diff --git a/api/handlers/profile.go b/api/handlers/profile.go index 5ebe38f84..a4b31a3f4 100644 --- a/api/handlers/profile.go +++ b/api/handlers/profile.go @@ -18,7 +18,7 @@ type ProfileHandlers struct { } func NewProfileHandlers(log logging.Logger, r repo.Repo) *ProfileHandlers { - req := core.NewProfileRequests(r) + req := core.NewProfileRequests(r, nil) h := ProfileHandlers{*req, log} return &h } @@ -38,7 +38,7 @@ func (h *ProfileHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request) func (h *ProfileHandlers) getProfileHandler(w http.ResponseWriter, r *http.Request) { args := true - res := &profile.Profile{} + res := &core.Profile{} if err := h.GetProfile(&args, res); err != nil { h.log.Infof("error getting profile: %s", err.Error()) util.WriteErrResponse(w, http.StatusInternalServerError, err) @@ -49,12 +49,12 @@ func (h *ProfileHandlers) getProfileHandler(w http.ResponseWriter, r *http.Reque } func (h *ProfileHandlers) saveProfileHandler(w http.ResponseWriter, r *http.Request) { - p := &profile.Profile{} + p := &core.Profile{} if err := json.NewDecoder(r.Body).Decode(p); err != nil { util.WriteErrResponse(w, http.StatusBadRequest, err) return } - res := &profile.Profile{} + res := &core.Profile{} if err := h.SaveProfile(p, res); err != nil { util.WriteErrResponse(w, http.StatusInternalServerError, err) return diff --git a/api/handlers/queries.go b/api/handlers/queries.go index 9d346d05b..29a13cac8 100644 --- a/api/handlers/queries.go +++ b/api/handlers/queries.go @@ -12,7 +12,7 @@ import ( ) func NewQueryHandlers(log logging.Logger, r repo.Repo) *QueryHandlers { - req := core.NewQueryRequests(r) + req := core.NewQueryRequests(r, nil) return &QueryHandlers{*req, log} } diff --git a/api/handlers/search.go b/api/handlers/search.go index c0b893f92..52bb48677 100644 --- a/api/handlers/search.go +++ b/api/handlers/search.go @@ -17,7 +17,7 @@ type SearchHandlers struct { } func NewSearchHandlers(log logging.Logger, r repo.Repo) *SearchHandlers { - req := core.NewSearchRequests(r) + req := core.NewSearchRequests(r, nil) return &SearchHandlers{*req, log} } diff --git a/cmd/profile.go b/cmd/profile.go new file mode 100644 index 000000000..8963dc1b3 --- /dev/null +++ b/cmd/profile.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "encoding/json" + "os" + + "github.com/qri-io/qri/core" + "github.com/spf13/cobra" +) + +var ( + setProfileFilepath string +) + +// profileCmd represents the profile command +var profileCmd = &cobra.Command{ + Use: "profile", + Short: "show or edit user profile information", +} + +var profileGetCmd = &cobra.Command{ + Use: "get", + Short: "get profile info", + Run: func(cmd *cobra.Command, args []string) { + r, err := ProfileRequests(false) + ExitIfErr(err) + + in := true + res := &core.Profile{} + err = r.GetProfile(&in, res) + ExitIfErr(err) + + data, err := json.MarshalIndent(res, "", " ") + ExitIfErr(err) + PrintSuccess(string(data)) + }, +} + +var profileSetCmd = &cobra.Command{ + Use: "set", + Short: "add peers to the profile list", + Run: func(cmd *cobra.Command, args []string) { + var ( + dataFile *os.File + err error + ) + + r, err := ProfileRequests(false) + ExitIfErr(err) + + dataFile, err = loadFileIfPath(setProfileFilepath) + ExitIfErr(err) + + p := &core.Profile{} + err = json.NewDecoder(dataFile).Decode(p) + ExitIfErr(err) + + res := &core.Profile{} + err = r.SaveProfile(p, res) + ExitIfErr(err) + + data, err := json.MarshalIndent(res, "", " ") + ExitIfErr(err) + PrintSuccess(string(data)) + }, +} + +func init() { + profileSetCmd.Flags().StringVarP(&setProfileFilepath, "file", "f", "", "json file to update profile info") + + profileCmd.AddCommand(profileGetCmd) + profileCmd.AddCommand(profileSetCmd) + RootCmd.AddCommand(profileCmd) +} diff --git a/cmd/queries.go b/cmd/queries.go index c3d824157..351601f07 100644 --- a/cmd/queries.go +++ b/cmd/queries.go @@ -14,11 +14,12 @@ var queriesCmd = &cobra.Command{ Long: ``, Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - req := core.NewQueryRequests(GetRepo(false)) + req, err := QueryRequests(false) + ExitIfErr(err) p := core.NewListParams("-created", pageNum, pageSize) res := []*repo.DatasetRef{} - err := req.List(&p, &res) + err = req.List(&p, &res) ExitIfErr(err) for i, q := range res { diff --git a/cmd/repo.go b/cmd/repo.go index 513287cff..4de25c1bb 100644 --- a/cmd/repo.go +++ b/cmd/repo.go @@ -48,6 +48,30 @@ func DatasetRequests(online bool) (*core.DatasetRequests, error) { return core.NewDatasetRequests(r, cli), nil } +func QueryRequests(online bool) (*core.QueryRequests, error) { + r, cli, err := RepoOrClient(online) + if err != nil { + return nil, err + } + return core.NewQueryRequests(r, cli), nil +} + +func ProfileRequests(online bool) (*core.ProfileRequests, error) { + r, cli, err := RepoOrClient(online) + if err != nil { + return nil, err + } + return core.NewProfileRequests(r, cli), nil +} + +func SearchRequests(online bool) (*core.SearchRequests, error) { + r, cli, err := RepoOrClient(online) + if err != nil { + return nil, err + } + return core.NewSearchRequests(r, cli), nil +} + // RepoOrClient returns either a func RepoOrClient(online bool) (repo.Repo, *rpc.Client, error) { if fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { diff --git a/cmd/run.go b/cmd/run.go index 1b14ef74f..79bc3beb8 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -27,8 +27,8 @@ var runCmd = &cobra.Command{ ErrExit(fmt.Errorf("Please provide a query string to execute")) } - r := GetRepo(false) - req := core.NewQueryRequests(r) + req, err := QueryRequests(false) + ExitIfErr(err) format, err := dataset.ParseDataFormatString(cmd.Flag("format").Value.String()) if err != nil { diff --git a/cmd/search.go b/cmd/search.go index 760a98ed0..8887b3a83 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -24,7 +24,8 @@ var searchCmd = &cobra.Command{ ErrExit(fmt.Errorf("wrong number of arguments. expected qri search [query]")) } - req := core.NewSearchRequests(GetRepo(false)) + req, err := SearchRequests(false) + ExitIfErr(err) if searchCmdReindex { PrintInfo("building index...") @@ -44,7 +45,7 @@ var searchCmd = &cobra.Command{ } res := []*repo.DatasetRef{} - err := req.Search(p, &res) + err = req.Search(p, &res) ExitIfErr(err) outformat := cmd.Flag("format").Value.String() diff --git a/core/core.go b/core/core.go index 8702feecb..17cb1f257 100644 --- a/core/core.go +++ b/core/core.go @@ -17,11 +17,11 @@ func Receivers(node *p2p.QriNode) []CoreRequests { r := node.Repo return []CoreRequests{ NewDatasetRequests(r, nil), - NewHistoryRequests(r), - NewPeerRequests(r, node), - NewProfileRequests(r), - NewQueryRequests(r), - NewSearchRequests(r), + NewHistoryRequests(r, nil), + NewPeerRequests(node, nil), + NewProfileRequests(r, nil), + NewQueryRequests(r, nil), + NewSearchRequests(r, nil), } } diff --git a/core/history.go b/core/history.go index 2d5eef335..63c08a81a 100644 --- a/core/history.go +++ b/core/history.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "net/rpc" "github.com/ipfs/go-datastore" "github.com/qri-io/dataset/dsfs" @@ -10,11 +11,16 @@ import ( type HistoryRequests struct { repo repo.Repo + cli *rpc.Client } func (d HistoryRequests) CoreRequestsName() string { return "history" } -func NewHistoryRequests(r repo.Repo) *HistoryRequests { +func NewHistoryRequests(r repo.Repo, cli *rpc.Client) *HistoryRequests { + if r != nil && cli != nil { + panic(fmt.Errorf("both repo and client supplied to NewHistoryRequests")) + } + return &HistoryRequests{ repo: r, } @@ -26,6 +32,10 @@ type LogParams struct { } func (d *HistoryRequests) Log(params *LogParams, res *[]*repo.DatasetRef) (err error) { + if d.cli != nil { + return d.cli.Call("HistoryRequests.Log", params, res) + } + log := []*repo.DatasetRef{} limit := params.Limit ref := &repo.DatasetRef{Path: params.Path} diff --git a/core/peers.go b/core/peers.go index 0f8ccad62..6b565da3e 100644 --- a/core/peers.go +++ b/core/peers.go @@ -3,6 +3,7 @@ package core import ( "encoding/json" "fmt" + "net/rpc" "github.com/ipfs/go-datastore/query" "github.com/qri-io/qri/p2p" @@ -12,30 +13,39 @@ import ( peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" ) -func NewPeerRequests(r repo.Repo, node *p2p.QriNode) *PeerRequests { +type PeerRequests struct { + qriNode *p2p.QriNode + cli *rpc.Client +} + +func NewPeerRequests(node *p2p.QriNode, cli *rpc.Client) *PeerRequests { + if node != nil && cli != nil { + panic(fmt.Errorf("both node and client supplied to NewPeerRequests")) + } + return &PeerRequests{ - repo: r, qriNode: node, + cli: cli, } } -type PeerRequests struct { - repo repo.Repo - qriNode *p2p.QriNode -} - func (d PeerRequests) CoreRequestsName() string { return "peers" } func (d *PeerRequests) List(p *ListParams, res *[]*profile.Profile) error { + if d.cli != nil { + return d.cli.Call("PeerRequests.List", p, res) + } + + r := d.qriNode.Repo replies := make([]*profile.Profile, p.Limit) i := 0 - user, err := d.repo.Profile() + user, err := r.Profile() if err != nil { return err } - ps, err := repo.QueryPeers(d.repo.Peers(), query.Query{}) + ps, err := repo.QueryPeers(r.Peers(), query.Query{}) if err != nil { return fmt.Errorf("error querying peers: %s", err.Error()) } @@ -56,16 +66,24 @@ func (d *PeerRequests) List(p *ListParams, res *[]*profile.Profile) error { } func (d *PeerRequests) ConnectedPeers(limit *int, peers *[]string) error { + if d.cli != nil { + return d.cli.Call("PeerRequests.ConnectedPeers", limit, peers) + } + *peers = d.qriNode.ConnectedPeers() return nil } func (d *PeerRequests) ConnectToPeer(pid *peer.ID, res *profile.Profile) error { + if d.cli != nil { + return d.cli.Call("PeerRequests.ConnectToPeer", pid, res) + } + if err := d.qriNode.ConnectToPeer(*pid); err != nil { return fmt.Errorf("error connecting to peer: %s", err.Error()) } - profile, err := d.repo.Peers().GetPeer(*pid) + profile, err := d.qriNode.Repo.Peers().GetPeer(*pid) if err != nil { return fmt.Errorf("error getting peer profile: %s", err.Error()) } @@ -75,6 +93,10 @@ func (d *PeerRequests) ConnectToPeer(pid *peer.ID, res *profile.Profile) error { } func (d *PeerRequests) Get(p *GetParams, res *profile.Profile) error { + if d.cli != nil { + return d.cli.Call("PeerRequests.Get", p, res) + } + // TODO - restore // peers, err := d.repo.Peers() // if err != nil { @@ -104,12 +126,16 @@ type NamespaceParams struct { } func (d *PeerRequests) GetNamespace(p *NamespaceParams, res *[]*repo.DatasetRef) error { + if d.cli != nil { + return d.cli.Call("PeerRequests.GetNamespace", p, res) + } + id, err := peer.IDB58Decode(p.PeerId) if err != nil { return fmt.Errorf("error decoding peer Id: %s", err.Error()) } - profile, err := d.repo.Peers().GetPeer(id) + profile, err := d.qriNode.Repo.Peers().GetPeer(id) if err != nil || profile == nil { return err } diff --git a/core/profile.go b/core/profile.go index 92dc58aca..fb05673bc 100644 --- a/core/profile.go +++ b/core/profile.go @@ -1,10 +1,13 @@ package core import ( + "encoding/json" "fmt" "io" "io/ioutil" "net/http" + "net/rpc" + "time" "github.com/qri-io/cafs/memfs" "github.com/qri-io/qri/repo" @@ -13,31 +16,101 @@ import ( type ProfileRequests struct { repo repo.Repo + cli *rpc.Client } func (d ProfileRequests) CoreRequestsName() string { return "profile" } -func NewProfileRequests(r repo.Repo) *ProfileRequests { +func NewProfileRequests(r repo.Repo, cli *rpc.Client) *ProfileRequests { + if r != nil && cli != nil { + panic(fmt.Errorf("both repo and client supplied to NewProfileRequests")) + } + return &ProfileRequests{ repo: r, + cli: cli, } } -func (r *ProfileRequests) GetProfile(in *bool, res *profile.Profile) error { +type Profile struct { + Id string `json:"id"` + Created time.Time `json:"created,omitempty"` + Updated time.Time `json:"updated,omitempty"` + Username string `json:"username"` + Type profile.UserType `json:"type"` + Email string `json:"email"` + Name string `json:"name"` + Description string `json:"description"` + HomeUrl string `json:"homeUrl"` + Color string `json:"color"` + Thumb string `json:"thumb"` + Profile string `json:"profile"` + Poster string `json:"poster"` + Twitter string `json:"twitter"` +} + +func unmarshalProfile(p *Profile) (*profile.Profile, error) { + // silly workaround for gob encoding + data, err := json.Marshal(p) + if err != nil { + return nil, fmt.Errorf("err re-encoding json: %s", err.Error()) + } + + _p := &profile.Profile{} + if err := json.Unmarshal(data, _p); err != nil { + return nil, fmt.Errorf("error unmarshaling json: %s", err.Error()) + } + + return _p, nil +} + +func marshalProfile(p *profile.Profile) (*Profile, error) { + // silly workaround for gob encoding + data, err := json.Marshal(p) + if err != nil { + return nil, fmt.Errorf("err re-encoding json: %s", err.Error()) + } + + _p := &Profile{} + if err := json.Unmarshal(data, _p); err != nil { + return nil, fmt.Errorf("error unmarshaling json: %s", err.Error()) + } + + return _p, nil +} + +func (r *ProfileRequests) GetProfile(in *bool, res *Profile) error { + if r.cli != nil { + return r.cli.Call("ProfileRequests.GetProfile", in, res) + } + profile, err := r.repo.Profile() if err != nil { return err } - *res = *profile + + _p, err := marshalProfile(profile) + if err != nil { + return err + } + *res = *_p return nil } -func (r *ProfileRequests) SaveProfile(p *profile.Profile, res *profile.Profile) error { +func (r *ProfileRequests) SaveProfile(p *Profile, res *Profile) error { + if r.cli != nil { + return r.cli.Call("ProfileRequests.SaveProfile", p, res) + } if p == nil { return fmt.Errorf("profile required for update") } - if err := r.repo.SaveProfile(p); err != nil { + _p, err := unmarshalProfile(p) + if err != nil { + return err + } + + if err := r.repo.SaveProfile(_p); err != nil { return err } @@ -46,7 +119,12 @@ func (r *ProfileRequests) SaveProfile(p *profile.Profile, res *profile.Profile) return err } - *res = *profile + p2, err := marshalProfile(profile) + if err != nil { + return err + } + + *res = *p2 return nil } @@ -58,6 +136,10 @@ type FileParams struct { } func (r *ProfileRequests) SetProfilePhoto(p *FileParams, res *profile.Profile) error { + if r.cli != nil { + return r.cli.Call("ProfileRequests.SetProfilePhoto", p, res) + } + if p.Data == nil { return fmt.Errorf("file is required") } @@ -99,6 +181,10 @@ func (r *ProfileRequests) SetProfilePhoto(p *FileParams, res *profile.Profile) e } func (r *ProfileRequests) SetPosterPhoto(p *FileParams, res *profile.Profile) error { + if r.cli != nil { + return r.cli.Call("ProfileRequests.SetPosterPhoto", p, res) + } + if p.Data == nil { return fmt.Errorf("file is required") } diff --git a/core/queries.go b/core/queries.go index f99359413..8309694c7 100644 --- a/core/queries.go +++ b/core/queries.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "net/rpc" "time" "github.com/ipfs/go-datastore" @@ -14,17 +15,27 @@ import ( type QueryRequests struct { repo repo.Repo + cli *rpc.Client } func (d QueryRequests) CoreRequestsName() string { return "queries" } -func NewQueryRequests(r repo.Repo) *QueryRequests { +func NewQueryRequests(r repo.Repo, cli *rpc.Client) *QueryRequests { + if r != nil && cli != nil { + panic(fmt.Errorf("both repo and client supplied to NewQueryRequests")) + } + return &QueryRequests{ repo: r, + cli: cli, } } func (d *QueryRequests) List(p *ListParams, res *[]*repo.DatasetRef) error { + if d.cli != nil { + return d.cli.Call("QueryRequests.List", p, res) + } + results, err := d.repo.GetQueryLogs(p.Limit, p.Offset) if err != nil { return fmt.Errorf("error getting query logs: %s", err.Error()) @@ -57,7 +68,11 @@ type GetQueryParams struct { } func (d *QueryRequests) Get(p *GetQueryParams, res *dataset.Dataset) error { - // TODO - huh? do we even need to load queries + if d.cli != nil { + return d.cli.Call("QueryRequests.Get", p, res) + } + + // TODO - huh? do we even need to load query datasets? q, err := dsfs.LoadDataset(d.repo.Store(), datastore.NewKey(p.Path)) if err != nil { return fmt.Errorf("error loading dataset: %s", err.Error()) @@ -75,6 +90,10 @@ type RunParams struct { } func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { + if r.cli != nil { + return r.cli.Call("QueryRequests.Run", p, res) + } + var ( store = r.repo.Store() transform *dataset.Transform @@ -237,6 +256,10 @@ type DatasetQueriesParams struct { } func (r *QueryRequests) DatasetQueries(p *DatasetQueriesParams, res *[]*repo.DatasetRef) error { + if r.cli != nil { + return r.cli.Call("QueryRequests.DatasetQueries", p, res) + } + if p.Path == "" { return fmt.Errorf("path is required") } diff --git a/core/search.go b/core/search.go index d35141984..b0e2f1c8a 100644 --- a/core/search.go +++ b/core/search.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "net/rpc" "github.com/qri-io/cafs" "github.com/qri-io/qri/repo" @@ -12,18 +13,27 @@ type SearchRequests struct { store cafs.Filestore repo repo.Repo // node *p2p.QriNode + cli *rpc.Client } func (d SearchRequests) CoreRequestsName() string { return "search" } -func NewSearchRequests(r repo.Repo) *SearchRequests { +func NewSearchRequests(r repo.Repo, cli *rpc.Client) *SearchRequests { + if r != nil && cli != nil { + panic(fmt.Errorf("both repo and client supplied to NewSearchRequests")) + } + return &SearchRequests{ repo: r, // node: node, + cli: cli, } } func (d *SearchRequests) Search(p *repo.SearchParams, res *[]*repo.DatasetRef) error { + if d.cli != nil { + return d.cli.Call("SearchRequests.Search", p, res) + } // if d.node != nil { // r, err := d.node.Search(p.Query, p.Limit, p.Offset) // if err != nil { @@ -49,6 +59,10 @@ type ReindexSearchParams struct { } func (d *SearchRequests) Reindex(p *ReindexSearchParams, done *bool) error { + if d.cli != nil { + return d.cli.Call("SearchRequests.Reindex", p, done) + } + if fsr, ok := d.repo.(*fs_repo.Repo); ok { err := fsr.UpdateSearchIndex(d.repo.Store()) if err != nil { From f3dea07d3847e4250271bb9bab18a0ef499fb235 Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 12 Dec 2017 10:22:48 -0500 Subject: [PATCH 07/10] fix: restoring tests, cleaning up post-ds.Transform refactor ) --- api/handlers/profile.go | 5 +- cmd/list.go | 19 ++++++- cmd/run.go | 3 +- core/history_test.go | 2 +- core/peers_test.go | 2 +- core/profile.go | 17 +++++- core/profile_test.go | 42 +++++++------- core/queries.go | 2 +- core/queries_test.go | 123 ++++++++++++++++++---------------------- repo/graph.go | 4 +- 10 files changed, 116 insertions(+), 103 deletions(-) diff --git a/api/handlers/profile.go b/api/handlers/profile.go index a4b31a3f4..c06ada2ab 100644 --- a/api/handlers/profile.go +++ b/api/handlers/profile.go @@ -8,7 +8,6 @@ import ( "github.com/qri-io/qri/core" "github.com/qri-io/qri/logging" "github.com/qri-io/qri/repo" - "github.com/qri-io/qri/repo/profile" ) // ProfileHandlers wraps a requests struct to interface with http.HandlerFunc @@ -91,7 +90,7 @@ func (h *ProfileHandlers) setProfilePhotoHandler(w http.ResponseWriter, r *http. } } - res := &profile.Profile{} + res := &core.Profile{} if err := h.SetProfilePhoto(p, res); err != nil { h.log.Infof("error initializing dataset: %s", err.Error()) util.WriteErrResponse(w, http.StatusInternalServerError, err) @@ -128,7 +127,7 @@ func (h *ProfileHandlers) setPosterHandler(w http.ResponseWriter, r *http.Reques } } - res := &profile.Profile{} + res := &core.Profile{} if err := h.SetPosterPhoto(p, res); err != nil { h.log.Infof("error initializing dataset: %s", err.Error()) util.WriteErrResponse(w, http.StatusInternalServerError, err) diff --git a/cmd/list.go b/cmd/list.go index 26336102c..e63df9b4d 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -4,9 +4,15 @@ import ( "encoding/json" "fmt" "github.com/qri-io/dataset" + "github.com/qri-io/qri/core" + "github.com/qri-io/qri/repo" "github.com/spf13/cobra" ) +var ( + dsListLimit, dsListOffset int +) + var datasetListCmd = &cobra.Command{ Use: "list", Aliases: []string{"ls"}, @@ -14,11 +20,18 @@ var datasetListCmd = &cobra.Command{ Long: ``, Run: func(cmd *cobra.Command, args []string) { // TODO - add limit & offset params - refs, err := GetRepo(false).Namespace(100, 0) + r, err := DatasetRequests(false) ExitIfErr(err) - outformat := cmd.Flag("format").Value.String() + p := &core.ListParams{ + Limit: dsListLimit, + Offset: dsListOffset, + } + refs := []*repo.DatasetRef{} + err = r.List(p, &refs) + ExitIfErr(err) + outformat := cmd.Flag("format").Value.String() switch outformat { case "": for _, ref := range refs { @@ -38,4 +51,6 @@ var datasetListCmd = &cobra.Command{ func init() { RootCmd.AddCommand(datasetListCmd) datasetListCmd.Flags().StringP("format", "f", "", "set output format [json]") + datasetListCmd.Flags().IntVarP(&dsListLimit, "limit", "l", 25, "limit results, default 25") + datasetListCmd.Flags().IntVarP(&dsListOffset, "offset", "o", 0, "offset results, default 0") } diff --git a/cmd/run.go b/cmd/run.go index 79bc3beb8..65f2d559e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -27,8 +27,9 @@ var runCmd = &cobra.Command{ ErrExit(fmt.Errorf("Please provide a query string to execute")) } - req, err := QueryRequests(false) + r, cli, err := RepoOrClient(false) ExitIfErr(err) + req := core.NewQueryRequests(r, cli) format, err := dataset.ParseDataFormatString(cmd.Flag("format").Value.String()) if err != nil { diff --git a/core/history_test.go b/core/history_test.go index ecfd82fa7..47cedfe13 100644 --- a/core/history_test.go +++ b/core/history_test.go @@ -30,7 +30,7 @@ func TestHistoryRequestsLog(t *testing.T) { {&LogParams{Path: path}, []*repo.DatasetRef{&repo.DatasetRef{Path: path}}, ""}, } - req := NewHistoryRequests(mr) + req := NewHistoryRequests(mr, nil) for i, c := range cases { got := []*repo.DatasetRef{} err := req.Log(c.p, &got) diff --git a/core/peers_test.go b/core/peers_test.go index a9a7f8d69..97786338e 100644 --- a/core/peers_test.go +++ b/core/peers_test.go @@ -28,7 +28,7 @@ func TestPeerRequestsList(t *testing.T) { } // TODO - need to upgrade this to include a mock node - req := NewPeerRequests(mr, &p2p.QriNode{}) + req := NewPeerRequests(&p2p.QriNode{Repo: mr}, nil) for i, c := range cases { got := []*profile.Profile{} err := req.List(c.p, &got) diff --git a/core/profile.go b/core/profile.go index fb05673bc..a46839841 100644 --- a/core/profile.go +++ b/core/profile.go @@ -135,7 +135,7 @@ type FileParams struct { Data io.Reader // reader of structured data. either Url or Data is required } -func (r *ProfileRequests) SetProfilePhoto(p *FileParams, res *profile.Profile) error { +func (r *ProfileRequests) SetProfilePhoto(p *FileParams, res *Profile) error { if r.cli != nil { return r.cli.Call("ProfileRequests.SetProfilePhoto", p, res) } @@ -176,11 +176,16 @@ func (r *ProfileRequests) SetProfilePhoto(p *FileParams, res *profile.Profile) e return fmt.Errorf("error saving profile: %s", err.Error()) } - *res = *pro + _p, err := marshalProfile(pro) + if err != nil { + return err + } + + *res = *_p return nil } -func (r *ProfileRequests) SetPosterPhoto(p *FileParams, res *profile.Profile) error { +func (r *ProfileRequests) SetPosterPhoto(p *FileParams, res *Profile) error { if r.cli != nil { return r.cli.Call("ProfileRequests.SetPosterPhoto", p, res) } @@ -219,5 +224,11 @@ func (r *ProfileRequests) SetPosterPhoto(p *FileParams, res *profile.Profile) er return fmt.Errorf("error saving profile: %s", err.Error()) } + _p, err := marshalProfile(pro) + if err != nil { + return err + } + + *res = *_p return nil } diff --git a/core/profile_test.go b/core/profile_test.go index 20bd5fd6d..a8d14fcb4 100644 --- a/core/profile_test.go +++ b/core/profile_test.go @@ -26,9 +26,9 @@ func TestProfileRequestsGet(t *testing.T) { return } - req := NewProfileRequests(mr) + req := NewProfileRequests(mr, nil) for i, c := range cases { - got := &profile.Profile{} + got := &Profile{} err := req.GetProfile(&c.in, got) if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { @@ -40,12 +40,12 @@ func TestProfileRequestsGet(t *testing.T) { func TestProfileRequestsSave(t *testing.T) { cases := []struct { - p *profile.Profile - res *profile.Profile + p *Profile + res *Profile err string }{ {nil, nil, "profile required for update"}, - {&profile.Profile{}, nil, ""}, + {&Profile{}, nil, ""}, // TODO - moar tests } @@ -55,9 +55,9 @@ func TestProfileRequestsSave(t *testing.T) { return } - req := NewProfileRequests(mr) + req := NewProfileRequests(mr, nil) for i, c := range cases { - got := &profile.Profile{} + got := &Profile{} err := req.SaveProfile(c.p, got) if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { @@ -73,9 +73,9 @@ func TestProfileRequestsSetProfilePhoto(t *testing.T) { respath datastore.Key err string }{ - {"", datastore.Key{}, "file is required"}, - {"testdata/ink_big_photo.jpg", datastore.Key{}, "file size too large. max size is 250kb"}, - {"testdata/q_bang.svg", datastore.Key{}, "invalid file format. only .jpg & .png images allowed"}, + {"", datastore.NewKey(""), "file is required"}, + {"testdata/ink_big_photo.jpg", datastore.NewKey(""), "file size too large. max size is 250kb"}, + {"testdata/q_bang.svg", datastore.NewKey(""), "invalid file format. only .jpg & .png images allowed"}, {"testdata/rico_400x400.jpg", datastore.NewKey("/map/QmRdexT18WuAKVX3vPusqmJTWLeNSeJgjmMbaF5QLGHna1"), ""}, } @@ -85,7 +85,7 @@ func TestProfileRequestsSetProfilePhoto(t *testing.T) { return } - req := NewProfileRequests(mr) + req := NewProfileRequests(mr, nil) for i, c := range cases { p := &FileParams{} if c.infile != "" { @@ -98,15 +98,15 @@ func TestProfileRequestsSetProfilePhoto(t *testing.T) { p.Data = r } - res := &profile.Profile{} + res := &Profile{} err := req.SetProfilePhoto(p, res) if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { t.Errorf("case %d error mismatch. expected: %s, got: %s", i, c.err, err.Error()) continue } - if !c.respath.Equal(res.Profile) { - t.Errorf("case %d profile hash mismatch. expected: %s, got: %s", i, c.respath.String(), res.Profile.String()) + if !c.respath.Equal(datastore.NewKey(res.Profile)) { + t.Errorf("case %d profile hash mismatch. expected: %s, got: %s", i, c.respath.String(), res.Profile) continue } } @@ -118,9 +118,9 @@ func TestProfileRequestsSetPosterPhoto(t *testing.T) { respath datastore.Key err string }{ - {"", datastore.Key{}, "file is required"}, - {"testdata/ink_big_photo.jpg", datastore.Key{}, "file size too large. max size is 250kb"}, - {"testdata/q_bang.svg", datastore.Key{}, "invalid file format. only .jpg & .png images allowed"}, + {"", datastore.NewKey(""), "file is required"}, + {"testdata/ink_big_photo.jpg", datastore.NewKey(""), "file size too large. max size is 250kb"}, + {"testdata/q_bang.svg", datastore.NewKey(""), "invalid file format. only .jpg & .png images allowed"}, {"testdata/rico_poster_1500x500.jpg", datastore.NewKey("/map/QmdJgfxj4rocm88PLeEididS7V2cc9nQosA46RpvAnWvDL"), ""}, } @@ -130,7 +130,7 @@ func TestProfileRequestsSetPosterPhoto(t *testing.T) { return } - req := NewProfileRequests(mr) + req := NewProfileRequests(mr, nil) for i, c := range cases { p := &FileParams{} if c.infile != "" { @@ -143,15 +143,15 @@ func TestProfileRequestsSetPosterPhoto(t *testing.T) { p.Data = r } - res := &profile.Profile{} + res := &Profile{} err := req.SetProfilePhoto(p, res) if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { t.Errorf("case %d error mismatch. expected: %s, got: %s", i, c.err, err.Error()) continue } - if !c.respath.Equal(res.Profile) { - t.Errorf("case %d profile hash mismatch. expected: %s, got: %s", i, c.respath.String(), res.Profile.String()) + if !c.respath.Equal(datastore.NewKey(res.Profile)) { + t.Errorf("case %d profile hash mismatch. expected: %s, got: %s", i, c.respath.String(), res.Profile) continue } } diff --git a/core/queries.go b/core/queries.go index 8309694c7..a1169195f 100644 --- a/core/queries.go +++ b/core/queries.go @@ -166,7 +166,7 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { if err != nil { return fmt.Errorf("formatting error: %s", err.Error()) } - qpath, err := dsfs.SaveTransform(store, abst, false) + qpath, err := dsfs.SaveAbstractTransform(store, abst, false) if err != nil { return fmt.Errorf("error calculating query hash: %s", err.Error()) } diff --git a/core/queries_test.go b/core/queries_test.go index aeea58a99..b8c3bea44 100644 --- a/core/queries_test.go +++ b/core/queries_test.go @@ -2,7 +2,6 @@ package core import ( "encoding/json" - "fmt" "testing" "github.com/qri-io/dataset" @@ -17,7 +16,7 @@ func TestList(t *testing.T) { t.Errorf("error allocating test repo: %s", err.Error()) return } - req := NewQueryRequests(mr) + req := NewQueryRequests(mr, nil) if req == nil { t.Errorf("error: expected non-nil result from NewQueryRequests()") return @@ -48,7 +47,7 @@ func TestGet(t *testing.T) { t.Errorf("error allocating test repo: %s", err.Error()) return } - req := NewQueryRequests(mr) + req := NewQueryRequests(mr, nil) if req == nil { t.Errorf("error: expected non-nil result from NewQueryRequests()") @@ -81,7 +80,7 @@ func TestRun(t *testing.T) { return } - req := NewQueryRequests(mr) + req := NewQueryRequests(mr, nil) if req == nil { t.Errorf("error: expected non-nil result from NewQueryRequests()") @@ -108,7 +107,6 @@ func TestRun(t *testing.T) { } if c.err == "" { - fmt.Println("path:", got.Path.String()) df, err := mr.Store().Get(got.Path) if err != nil { t.Errorf("case %d error getting dataset path: %s: %s", i, got.Path.String(), err.Error()) @@ -130,72 +128,61 @@ func TestRun(t *testing.T) { if !ds.Structure.IsEmpty() { t.Errorf("expected stored dataset.Structure to be a reference") } - if !ds.AbstractStructure.IsEmpty() { - t.Errorf("expected stored dataset.AbstractStructure to be a reference") + if !ds.Abstract.IsEmpty() { + t.Errorf("expected stored dataset.Abstract to be a reference") } } } } -// TODO - RESTORE BEFORE MERGING -// func TestDatasetQueries(t *testing.T) { -// mr, err := testrepo.NewTestRepo() -// if err != nil { -// t.Errorf("error allocating test repo: %s", err.Error()) -// return -// } - -// req := NewQueryRequests(mr) - -// path, err := mr.GetPath("movies") -// if err != nil { -// t.Errorf("errog getting path for 'movies' dataset: %s", err.Error()) -// return -// } - -// // ns, err := mr.Namespace(30, 0) -// // if err != nil { -// // t.Errorf("error getting repo namespace: %s", err.Error()) -// // return -// // } - -// // for _, n := range ns { -// // fmt.Println(n) -// // } - -// qres := &repo.DatasetRef{} -// if err = req.Run(&RunParams{ -// Dataset: &dataset.Dataset{ -// QueryString: "select * from movies", -// }}, qres); err != nil { -// t.Errorf("error running query: %s", err.Error()) -// return -// } - -// cases := []struct { -// p *DatasetQueriesParams -// res []*repo.DatasetRef -// err string -// }{ -// {&DatasetQueriesParams{}, []*repo.DatasetRef{}, "path is required"}, -// {&DatasetQueriesParams{Path: path.String()}, []*repo.DatasetRef{&repo.DatasetRef{}}, ""}, -// // TODO: add more tests -// } - -// for i, c := range cases { -// got := []*repo.DatasetRef{} -// err := req.DatasetQueries(c.p, &got) -// if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { -// t.Errorf("case %d error mismatch: expected: %s, got: %s", i, c.err, err) -// continue -// } - -// // fmt.Println(got) - -// if len(c.res) != len(got) { -// t.Errorf("case %d returned wrong number of responses. exepected: %d, got %d", i, len(c.res), len(got)) -// continue -// } -// } -// } +func TestDatasetQueries(t *testing.T) { + mr, err := testrepo.NewTestRepo() + if err != nil { + t.Errorf("error allocating test repo: %s", err.Error()) + return + } + + req := NewQueryRequests(mr, nil) + + path, err := mr.GetPath("movies") + if err != nil { + t.Errorf("errog getting path for 'movies' dataset: %s", err.Error()) + return + } + + qres := &repo.DatasetRef{} + if err = req.Run(&RunParams{ + Dataset: &dataset.Dataset{ + QueryString: "select * from movies", + }}, qres); err != nil { + t.Errorf("error running query: %s", err.Error()) + return + } + + cases := []struct { + p *DatasetQueriesParams + res []*repo.DatasetRef + err string + }{ + {&DatasetQueriesParams{}, []*repo.DatasetRef{}, "path is required"}, + {&DatasetQueriesParams{Path: path.String()}, []*repo.DatasetRef{&repo.DatasetRef{}}, ""}, + // TODO: ALWAYS MOAR TESTS. OM NOM NOM FEED THE TEST MONSTER. + } + + for i, c := range cases { + got := []*repo.DatasetRef{} + err := req.DatasetQueries(c.p, &got) + if !(err == nil && c.err == "" || err != nil && err.Error() == c.err) { + t.Errorf("case %d error mismatch: expected: %s, got: %s", i, c.err, err) + continue + } + + // fmt.Println(got) + + if len(c.res) != len(got) { + t.Errorf("case %d returned wrong number of responses. exepected: %d, got %d", i, len(c.res), len(got)) + continue + } + } +} diff --git a/repo/graph.go b/repo/graph.go index 79f9dbc32..33c60852f 100644 --- a/repo/graph.go +++ b/repo/graph.go @@ -145,10 +145,10 @@ func (nl NodeList) nodesFromDatasetRef(r Repo, ref *DatasetRef) *dsgraph.Node { root.AddLinks(dsgraph.Link{From: root, To: commit}) } - if ds.AbstractStructure != nil && ds.AbstractStructure.Path().String() != "" { + if ds.Abstract != nil && ds.Abstract.Path().String() != "" { root.AddLinks(dsgraph.Link{ From: root, - To: nl.node(dsgraph.NtAbstStructure, ds.AbstractStructure.Path().String()), + To: nl.node(dsgraph.NtAbstDataset, ds.Abstract.Path().String()), }) } From 6010e9f6f18b06ee90e4fa29aa4a4cc99613682d Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 12 Dec 2017 15:19:34 -0500 Subject: [PATCH 08/10] fix: more work on settling Transform refactor --- core/queries.go | 101 +++++++++++++++++------------------------- repo/fs/query_logs.go | 94 ++++++++++----------------------------- repo/graph.go | 12 ++--- repo/mem_logs.go | 26 +++++++++-- repo/repo.go | 14 +++++- 5 files changed, 106 insertions(+), 141 deletions(-) diff --git a/core/queries.go b/core/queries.go index a1169195f..a67a0a33c 100644 --- a/core/queries.go +++ b/core/queries.go @@ -36,14 +36,16 @@ func (d *QueryRequests) List(p *ListParams, res *[]*repo.DatasetRef) error { return d.cli.Call("QueryRequests.List", p, res) } - results, err := d.repo.GetQueryLogs(p.Limit, p.Offset) + items, err := d.repo.ListQueryLogs(p.Limit, p.Offset) if err != nil { return fmt.Errorf("error getting query logs: %s", err.Error()) } - for _, ref := range results { - if ds, err := dsfs.LoadDataset(d.repo.Store(), ref.Path); err == nil { - ref.Dataset = ds + results := make([]*repo.DatasetRef, len(items)) + for i, item := range items { + results[i].Path = item.DatasetPath + if ds, err := dsfs.LoadDataset(d.repo.Store(), item.DatasetPath); err == nil { + results[i].Dataset = ds } } @@ -95,17 +97,16 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { } var ( - store = r.repo.Store() - transform *dataset.Transform - results []byte - err error - ds = p.Dataset + store = r.repo.Store() + abst *dataset.Transform + results []byte + err error + ds = p.Dataset ) if ds == nil { return fmt.Errorf("dataset is required") } - // fmt.Println("running query: %s", p.Dataset.QueryString) ds.Timestamp = time.Now() @@ -117,16 +118,6 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { } } - // if ds.QueryString == "" { - // ds.QueryString = q.Abstract.Statement - // } - - // TODO - make format output the parsed statement as well - // to avoid triple-parsing - // sqlstr, _, remap, err := sql.Format(ds.QueryString) - // if err != nil { - // return err - // } names, err := sql.StatementTableNames(q.Data) if err != nil { return fmt.Errorf("error getting statement table names: %s", err.Error()) @@ -148,67 +139,53 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { } } - // func PreparedQueryPath(fs cafs.Filestore, q *dataset.Query, opts *ExecOpt) (datastore.Key, error) { - // q2 := &dataset.Query{} - // q2.Assign(q) - // prep, err := Prepare(q2, opts) - // if err != nil { - // return datastore.NewKey(""), err - // } - // return dsfs.SaveQuery(fs, prep.q, false) - // } - q2 := &dataset.Transform{} q2.Assign(q) - _, abst, err := sql.Format(q, func(o *sql.ExecOpt) { + // _, abst, err = sql.Format(q, func(o *sql.ExecOpt) { + // o.Format = dataset.CSVDataFormat + // }) + // if err != nil { + // return fmt.Errorf("formatting error: %s", err.Error()) + // } + // qpath, err := dsfs.SaveAbstractTransform(store, abst, false) + fmt.Println("queries", q.Data, q2.Data) + qrpath, err := sql.QueryRecordPath(store, q2, func(o *sql.ExecOpt) { o.Format = dataset.CSVDataFormat }) - if err != nil { - return fmt.Errorf("formatting error: %s", err.Error()) - } - qpath, err := dsfs.SaveAbstractTransform(store, abst, false) if err != nil { return fmt.Errorf("error calculating query hash: %s", err.Error()) } - // fmt.Println(qpath.String()) - // atb, _ := abst.MarshalJSON() - // fmt.Println(string(atb)) - - // qpath, err := sql.PreparedQueryPath(r.repo.Store(), q, &sql.ExecOpt{Format: dataset.CSVDataFormat}) - // if err != nil { - // return fmt.Errorf("error calculating query hash: %s", err.Error()) - // } - - if dsp, err := repo.DatasetForQuery(r.repo, qpath); err != nil && err != repo.ErrNotFound { + if qi, err := r.repo.QueryLogItem(&repo.QueryLogItem{Key: qrpath}); err != nil && err != repo.ErrNotFound { return fmt.Errorf("error checking for existing query: %s", err.Error()) } else if err != repo.ErrNotFound { - if ds, err := dsfs.LoadDataset(store, dsp); err == nil { - ref := &repo.DatasetRef{Name: p.SaveName, Path: dsp, Dataset: ds} - if err := r.repo.LogQuery(ref); err != nil { - return fmt.Errorf("error logging query to repo: %s", err.Error()) + if ds, err := dsfs.LoadDataset(store, qi.DatasetPath); err == nil { + // ref := &repo.QueryLogItem{Name: p.SaveName, Query: q.Data, Key: dsp, Dataset: dsp} + // if err := r.repo.LogQuery(ref); err != nil { + // return fmt.Errorf("error logging query to repo: %s", err.Error()) + // } + *res = repo.DatasetRef{ + Path: qi.DatasetPath, + Dataset: ds, } - *res = *ref return nil } } // TODO - detect data format from passed-in results structure - transform, results, err = sql.Exec(store, q, func(o *sql.ExecOpt) { + abst, results, err = sql.Exec(store, q, func(o *sql.ExecOpt) { o.Format = dataset.CSVDataFormat }) if err != nil { return fmt.Errorf("error executing query: %s", err.Error()) } - // tb, _ := transform.MarshalJSON() - // fmt.Println(string(tb)) - // TODO - move this into setting on the dataset outparam - ds.Structure = transform.Structure + ds.Structure = q.Structure ds.Length = len(results) ds.Transform = q - ds.AbstractTransform = transform + ds.AbstractTransform = abst + fmt.Printf("abst: %#v\n", abst) datakey, err := store.Put(memfs.NewMemfileBytes("data."+ds.Structure.Format.String(), results), false) if err != nil { @@ -232,15 +209,19 @@ func (r *QueryRequests) Run(p *RunParams, res *repo.DatasetRef) error { if err := dsfs.DerefDatasetStructure(store, ds); err != nil { return fmt.Errorf("error dereferencing dataset structure: %s", err.Error()) } - // fmt.Println("result query:", ds.AbstractTransform.Path()) if err := dsfs.DerefDatasetTransform(store, ds); err != nil { return fmt.Errorf("error dereferencing dataset query: %s", err.Error()) } - // fmt.Println(ds.AbstractTransform.Path().String()) ref := &repo.DatasetRef{Name: p.SaveName, Path: dspath, Dataset: ds} - - if err := r.repo.LogQuery(ref); err != nil { + item := &repo.QueryLogItem{ + Query: ds.QueryString, + Name: p.SaveName, + Key: qrpath, + DatasetPath: dspath, + Time: time.Now(), + } + if err := r.repo.LogQuery(item); err != nil { return fmt.Errorf("error logging query to repo: %s", err.Error()) } diff --git a/repo/fs/query_logs.go b/repo/fs/query_logs.go index fd721fd17..c7136ad26 100644 --- a/repo/fs/query_logs.go +++ b/repo/fs/query_logs.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "os" + "sort" "github.com/qri-io/cafs" "github.com/qri-io/qri/repo" @@ -20,16 +21,34 @@ func NewQueryLog(base string, file File, store cafs.Filestore) QueryLog { return QueryLog{basepath: basepath(base), file: file, store: store} } -func (ql QueryLog) LogQuery(ref *repo.DatasetRef) error { +func (ql QueryLog) LogQuery(item *repo.QueryLogItem) error { log, err := ql.logs() if err != nil { return err } - log = append([]*repo.DatasetRef{&repo.DatasetRef{Name: ref.Name, Path: ref.Path}}, log...) + log = append(log, item) + sort.Slice(log, func(i, j int) bool { return log[i].Time.Before(log[j].Time) }) return ql.saveFile(log, ql.file) } -func (ql QueryLog) GetQueryLogs(limit, offset int) ([]*repo.DatasetRef, error) { +func (ql QueryLog) QueryLogItem(q *repo.QueryLogItem) (*repo.QueryLogItem, error) { + log, err := ql.logs() + if err != nil { + return nil, err + } + + for _, item := range log { + if item.DatasetPath.Equal(q.DatasetPath) || + item.Query == q.Query || + item.Time.Equal(q.Time) || + item.Key.Equal(q.Key) { + return item, nil + } + } + return nil, repo.ErrNotFound +} + +func (ql QueryLog) ListQueryLogs(limit, offset int) ([]*repo.QueryLogItem, error) { logs, err := ql.logs() if err != nil { return nil, err @@ -46,73 +65,8 @@ func (ql QueryLog) GetQueryLogs(limit, offset int) ([]*repo.DatasetRef, error) { return logs[offset:stop], nil } -// func (r QueryLog) PutDataset(path datastore.Key, ds *dataset.Dataset) error { -// d, err := r.logs() -// if err != nil { -// return err -// } -// d[path.String()] = ds -// return r.saveFile(d, r.file) -// } - -// func (r QueryLog) PutQueryLog(logs []*repo.repo.DatasetRef) error { -// ds, err := r.logs() -// if err != nil { -// return err -// } -// for _, dr := range logs { -// ps := dr.Path.String() -// if ps != "" && dr.Dataset != nil { -// ds[ps] = dr.Dataset -// } -// } -// return r.saveFile(ds, r.file) -// } - -// func (r QueryLog) GetDataset(path datastore.Key) (*dataset.Dataset, error) { -// ds, err := r.logs() -// if err != nil { -// return nil, err -// } -// ps := path.String() -// for p, d := range ds { -// if ps == p { -// return d, nil -// } -// } -// if r.store != nil { -// return dsfs.LoadDataset(r.store, path) -// } - -// return nil, datastore.ErrNotFound -// } - -// func (r QueryLog) DeleteDataset(path datastore.Key) error { -// ds, err := r.logs() -// if err != nil { -// return err -// } -// delete(ds, path.String()) -// return r.saveFile(ds, r.file) -// } - -// func (r QueryLog) Query(q query.Query) (query.Results, error) { -// ds, err := r.logs() -// if err != nil { -// return nil, err -// } - -// re := make([]query.Entry, 0, len(ds)) -// for path, d := range ds { -// re = append(re, query.Entry{Key: path, Value: d}) -// } -// res := query.ResultsWithEntries(q, re) -// res = query.NaiveQueryApply(q, res) -// return res, nil -// } - -func (r *QueryLog) logs() ([]*repo.DatasetRef, error) { - ds := []*repo.DatasetRef{} +func (r *QueryLog) logs() ([]*repo.QueryLogItem, error) { + ds := []*repo.QueryLogItem{} data, err := ioutil.ReadFile(r.filepath(r.file)) if err != nil { if os.IsNotExist(err) { diff --git a/repo/graph.go b/repo/graph.go index 33c60852f..d24d6de49 100644 --- a/repo/graph.go +++ b/repo/graph.go @@ -259,18 +259,20 @@ func WalkRepoDatasets(r Repo, visit func(depth int, ref *DatasetRef, err error) // TODO - make properly parallel go func() { - refs, err := r.GetQueryLogs(1000, 0) + items, err := r.ListQueryLogs(1000, 0) if err != nil { done <- err } - for _, ref := range refs { - ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path) + for _, item := range items { + ref := &DatasetRef{Path: item.DatasetPath} + + ref.Dataset, err = dsfs.LoadDatasetRefs(store, item.DatasetPath) // TODO - remove this once loading is more consistent. if err != nil { - ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path) + ref.Dataset, err = dsfs.LoadDatasetRefs(store, item.DatasetPath) } if err != nil { - ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path) + ref.Dataset, err = dsfs.LoadDatasetRefs(store, item.DatasetPath) } kontinue, err := visit(0, ref, err) diff --git a/repo/mem_logs.go b/repo/mem_logs.go index 04d77c90f..6c4b941f4 100644 --- a/repo/mem_logs.go +++ b/repo/mem_logs.go @@ -1,13 +1,31 @@ package repo -type MemQueryLog []*DatasetRef +import ( + "sort" +) -func (ql *MemQueryLog) LogQuery(ref *DatasetRef) error { - *ql = append(*ql, &DatasetRef{Name: ref.Name, Path: ref.Path}) +type MemQueryLog []*QueryLogItem + +func (ql *MemQueryLog) LogQuery(item *QueryLogItem) error { + logs := append(*ql, item) + sort.Slice(logs, func(i, j int) bool { return logs[i].Time.Before(logs[j].Time) }) + *ql = logs return nil } -func (ql MemQueryLog) GetQueryLogs(limit, offset int) ([]*DatasetRef, error) { +func (ql *MemQueryLog) QueryLogItem(q *QueryLogItem) (*QueryLogItem, error) { + for _, item := range *ql { + if item.DatasetPath.Equal(q.DatasetPath) || + item.Query == q.Query || + item.Time.Equal(q.Time) || + item.Key.Equal(q.Key) { + return item, nil + } + } + return nil, ErrNotFound +} + +func (ql MemQueryLog) ListQueryLogs(limit, offset int) ([]*QueryLogItem, error) { if offset > len(ql) { offset = len(ql) } diff --git a/repo/repo.go b/repo/repo.go index e160ae0bf..183771996 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -7,6 +7,7 @@ package repo import ( "fmt" + "time" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" @@ -105,10 +106,19 @@ type Datasets interface { Query(query.Query) (query.Results, error) } +type QueryLogItem struct { + Query string + Name string + Key datastore.Key + DatasetPath datastore.Key + Time time.Time +} + // QueryLog keeps logs type QueryLog interface { - LogQuery(*DatasetRef) error - GetQueryLogs(limit, offset int) ([]*DatasetRef, error) + LogQuery(*QueryLogItem) error + ListQueryLogs(limit, offset int) ([]*QueryLogItem, error) + QueryLogItem(q *QueryLogItem) (*QueryLogItem, error) } // SearchParams encapsulates parameters provided to Searchable.Search From 5ce7d382f40868cf2b99c2efbebf3ad7761c2d62 Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 12 Dec 2017 16:12:23 -0500 Subject: [PATCH 09/10] fix(cmd.queriesCmd): fix query listing command --- cmd/print.go | 2 +- core/queries.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/print.go b/cmd/print.go index 4f30fc18e..ce8d4214f 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -125,7 +125,7 @@ func PrintQuery(i int, r *repo.DatasetRef) { white := color.New(color.FgWhite).SprintFunc() cyan := color.New(color.FgCyan).SprintFunc() blue := color.New(color.FgBlue).SprintFunc() - fmt.Printf("%s:\t%s\n\t%s\n", cyan(i), white(r.Dataset.QueryString), blue(r.Path)) + fmt.Printf("%s:\t%s\n\t%s\n", cyan(i), white(r.Dataset.Transform.Data), blue(r.Path)) } func PrintResults(r *dataset.Structure, data []byte, format dataset.DataFormat) { diff --git a/core/queries.go b/core/queries.go index a67a0a33c..7a5c2325d 100644 --- a/core/queries.go +++ b/core/queries.go @@ -43,8 +43,9 @@ func (d *QueryRequests) List(p *ListParams, res *[]*repo.DatasetRef) error { results := make([]*repo.DatasetRef, len(items)) for i, item := range items { - results[i].Path = item.DatasetPath + results[i] = &repo.DatasetRef{Path: item.DatasetPath} if ds, err := dsfs.LoadDataset(d.repo.Store(), item.DatasetPath); err == nil { + results[i].Name = ds.Transform.Data results[i].Dataset = ds } } From 6017ea3108690915ee7b3024422f93d9d6defc4b Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 12 Dec 2017 16:41:55 -0500 Subject: [PATCH 10/10] fix(deps): fix outdated ipfs gx dep --- p2p/bootstrap.go | 2 +- p2p/node.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index 52f686616..4d310208b 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -5,9 +5,9 @@ import ( "math/rand" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" + math2 "gx/ipfs/QmViBzgruNUoLNBnXcx8YWbDNwV8MNGEGKkLo6JGetygdw/go-ipfs/thirdparty/math2" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" - math2 "gx/ipfs/QmdKL1GVaUaDVt3JUWiYQSLYRsJMym2KRWxsiXAeEU6pzX/go-ipfs/thirdparty/math2" ) // DefaultBootstrapAddresses follows the pattern of IPFS boostrapping off known "gateways". diff --git a/p2p/node.go b/p2p/node.go index b962310b7..afc8b36cc 100644 --- a/p2p/node.go +++ b/p2p/node.go @@ -10,12 +10,12 @@ import ( yamux "gx/ipfs/QmNWCEvi7bPRcvqAV8AKLGVNoQdArWi7NJayka2SM4XtRe/go-smux-yamux" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" + core "gx/ipfs/QmViBzgruNUoLNBnXcx8YWbDNwV8MNGEGKkLo6JGetygdw/go-ipfs/core" msmux "gx/ipfs/QmVniQJkdzLZaZwzwMdd3dJTvWiJ1DQEkreVy6hs6h7Vk5/go-smux-multistream" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" crypto "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" host "gx/ipfs/Qmc1XhrFEiSeBNn3mpfg6gEuYCt5im2gYmNVmncsvmpeAk/go-libp2p-host" - core "gx/ipfs/QmdKL1GVaUaDVt3JUWiYQSLYRsJMym2KRWxsiXAeEU6pzX/go-ipfs/core" swarm "gx/ipfs/QmdQFrFnPrKRQtpeHKjZ3cVNwxmGKKS2TvhJTuN9C9yduh/go-libp2p-swarm" discovery "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/discovery" bhost "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/host/basic"