Merge pull request bittorrent#269 from TRON-US/BTFS-1166

BTFS-1166

core/commands/guard.go

@@ -105,7 +105,7 @@ to the guard service.`,
  return err
  }
  for _, ni := range hosts {
- hostIDs = append(hostIDs, ni.NodeID)
+ hostIDs = append(hostIDs, ni.NodeId)
  }
  }
  if len(hostIDs) < len(shardHashes) {

core/commands/storage.go

@@ -9,26 +9,28 @@ import (
  "strings"
  "time"
 
- config "github.com/TRON-US/go-btfs-config"
  "github.com/TRON-US/go-btfs/core"
  "github.com/TRON-US/go-btfs/core/commands/cmdenv"
  "github.com/TRON-US/go-btfs/core/commands/storage"
  "github.com/TRON-US/go-btfs/core/corehttp/remote"
  "github.com/TRON-US/go-btfs/core/escrow"
  "github.com/TRON-US/go-btfs/core/hub"
- escrowPb "github.com/tron-us/go-btfs-common/protos/escrow"
- ledgerPb "github.com/tron-us/go-btfs-common/protos/ledger"
 
  cmds "github.com/TRON-US/go-btfs-cmds"
+ config "github.com/TRON-US/go-btfs-config"
  coreiface "github.com/TRON-US/interface-go-btfs-core"
  "github.com/TRON-US/interface-go-btfs-core/path"
+ "github.com/tron-us/go-btfs-common/crypto"
+ "github.com/tron-us/go-btfs-common/info"
+ escrowpb "github.com/tron-us/go-btfs-common/protos/escrow"
+ hubpb "github.com/tron-us/go-btfs-common/protos/hub"
+ ledgerpb "github.com/tron-us/go-btfs-common/protos/ledger"
+
  "github.com/gogo/protobuf/proto"
  cidlib "github.com/ipfs/go-cid"
  ds "github.com/ipfs/go-datastore"
  "github.com/libp2p/go-libp2p-core/peer"
  ma "github.com/multiformats/go-multiaddr"
- "github.com/tron-us/go-btfs-common/crypto"
- "github.com/tron-us/go-btfs-common/info"
 )
 
 const (
@@ -225,7 +227,7 @@ Receive proofs as collateral evidence after selected nodes agree to store the fi
  }
  // add host to retry queue
  host := &storage.HostNode{
- Identity: ni.NodeID,
+ Identity: ni.NodeId,
  RetryTimes: 0,
  FailTimes: 0,
  Price: price,
@@ -545,7 +547,7 @@ var storageUploadRecvContractCmd = &cmds.Command{
  if err != nil {
  return err
  }
- var contractRequest *escrowPb.EscrowContractRequest
+ var contractRequest *escrowpb.EscrowContractRequest
 
  if ss.GetCompleteContractNum() == len(ss.ChunkInfo) {
  contracts, price, err := storage.PrepareContractFromChunk(ss.ChunkInfo)
@@ -567,7 +569,7 @@ var storageUploadRecvContractCmd = &cmds.Command{
  },
 }
 
-func payFullToEscrow(response *escrowPb.SignedSubmitContractResult, configuration *config.Config) {
+func payFullToEscrow(response *escrowpb.SignedSubmitContractResult, configuration *config.Config) {
  privKeyStr := configuration.Identity.PrivKey
  payerPrivKey, err := crypto.ToPrivKey(privKeyStr)
  if err != nil {
@@ -843,7 +845,7 @@ func completePayment(chunkInfo *storage.Chunk, chunkHash string, ssID string, re
  storage.GlobalSession.Remove(ssID, chunkHash)
  return
  }
- var halfSignedChannelState ledgerPb.SignedChannelState
+ var halfSignedChannelState ledgerpb.SignedChannelState
  err := proto.Unmarshal(payment.SignedPayment, &halfSignedChannelState)
  if err != nil {
  log.Error(err)
@@ -1115,7 +1117,7 @@ Mode options include:
 }
 
 type HostInfoRes struct {
- Nodes []*info.Node
+ Nodes []*hubpb.Host
 }
 
 var storageHostsSyncCmd = &cmds.Command{
@@ -1161,7 +1163,7 @@ Mode options include:
 }
 
 func SyncHosts(ctx context.Context, node *core.IpfsNode, mode string) error {
- nodes, err := hub.QueryHub(node, mode)
+ nodes, err := hub.QueryHub(ctx, node, mode)
  if err != nil {
  return err
  }

core/commands/storage/hosts.go

@@ -8,7 +8,8 @@ import (
 
  "github.com/TRON-US/go-btfs/core"
  "github.com/TRON-US/go-btfs/core/hub"
- "github.com/tron-us/go-btfs-common/info"
+
+ hubpb "github.com/tron-us/go-btfs-common/protos/hub"
 
  ds "github.com/ipfs/go-datastore"
  "github.com/ipfs/go-datastore/query"
@@ -24,7 +25,7 @@ const (
 // GetHostsFromDatastore retrieves `num` hosts from the datastore, if not enough hosts are
 // available, return an error instead of partial return.
 // When num=0 it means unlimited.
-func GetHostsFromDatastore(ctx context.Context, node *core.IpfsNode, mode string, num int) ([]*info.Node, error) {
+func GetHostsFromDatastore(ctx context.Context, node *core.IpfsNode, mode string, num int) ([]*hubpb.Host, error) {
  // check mode: all = display everything
  if mode == hub.HubModeAll {
  mode = ""
@@ -39,17 +40,17 @@ func GetHostsFromDatastore(ctx context.Context, node *core.IpfsNode, mode string
  return nil, err
  }
  // Add as many hosts as available
- var hosts []*info.Node
+ var hosts []*hubpb.Host
  for r := range qr.Next() {
  if r.Error != nil {
  return nil, r.Error
  }
- var ni info.Node
- err := json.Unmarshal(r.Entry.Value, &ni)
+ var h hubpb.Host
+ err := json.Unmarshal(r.Entry.Value, &h)
  if err != nil {
  return nil, err
  }
- hosts = append(hosts, &ni)
+ hosts = append(hosts, &h)
  }
  // we can re-use hosts, but for higher availability, we choose to have the
  // greater than `num assumption
@@ -69,7 +70,7 @@ func newKeyHelper(kss ...string) ds.Key {
 
 // SaveHostsIntoDatastore overwrites (removes all existing) hosts and saves the updated
 // hosts according to mode.
-func SaveHostsIntoDatastore(ctx context.Context, node *core.IpfsNode, mode string, nodes []*info.Node) error {
+func SaveHostsIntoDatastore(ctx context.Context, node *core.IpfsNode, mode string, nodes []*hubpb.Host) error {
  rds := node.Repo.Datastore()
 
  // Dumb strategy right now: remove all existing and add the new ones
@@ -94,7 +95,7 @@ func SaveHostsIntoDatastore(ctx context.Context, node *core.IpfsNode, mode strin
  if err != nil {
  return err
  }
- err = rds.Put(newKeyHelper(HostStorePrefix, mode, "/", fmt.Sprintf("%04d", i), "/", ni.NodeID), b)
+ err = rds.Put(newKeyHelper(HostStorePrefix, mode, "/", fmt.Sprintf("%04d", i), "/", ni.NodeId), b)
  if err != nil {
  return err
  }

core/commands/storage/hosts_test.go

@@ -1,14 +1,16 @@
 package storage
 
 import (
+ "bytes"
  "context"
  "fmt"
  "testing"
 
+ unixtest "github.com/TRON-US/go-btfs/core/coreunix/test"
  "github.com/TRON-US/go-btfs/core/hub"
- "github.com/tron-us/go-btfs-common/info"
 
- unixtest "github.com/TRON-US/go-btfs/core/coreunix/test"
+ "github.com/gogo/protobuf/proto"
+ hubpb "github.com/tron-us/go-btfs-common/protos/hub"
 )
 
 func TestHostsSaveGet(t *testing.T) {
@@ -17,9 +19,9 @@ func TestHostsSaveGet(t *testing.T) {
  // test all possible modes
  for _, mode := range []string{hub.HubModeAll, hub.HubModeScore,
  hub.HubModeGeo, hub.HubModeRep, hub.HubModePrice, hub.HubModeSpeed} {
- var nodes []*info.Node
+ var nodes []*hubpb.Host
  for i := 0; i < 100; i++ {
- ni := &info.Node{NodeID: fmt.Sprintf("%s:node:%d", mode, i)}
+ ni := &hubpb.Host{NodeId: fmt.Sprintf("%s:node:%d", mode, i)}
  nodes = append(nodes, ni)
  }
  err := SaveHostsIntoDatastore(context.Background(), node, mode, nodes)
@@ -31,7 +33,15 @@ func TestHostsSaveGet(t *testing.T) {
  t.Fatal(err)
  }
  for i, sn := range stored {
- if *sn != *nodes[i] {
+ bs1, err := proto.Marshal(sn)
+ if err != nil {
+ t.Fatal(err)
+ }
+ bs2, err := proto.Marshal(nodes[i])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if bytes.Compare(bs1, bs2) != 0 {
  t.Fatal("stored nodes do not match saved nodes")
  }
  }

core/hub/sync.go

@@ -1,13 +1,12 @@
 package hub
 
 import (
- "encoding/json"
+ "context"
  "fmt"
- "net/http"
-
  "github.com/TRON-US/go-btfs/core"
 
- "github.com/tron-us/go-btfs-common/info"
+ hubpb "github.com/tron-us/go-btfs-common/protos/hub"
+ "github.com/tron-us/go-btfs-common/utils/grpc"
 )
 
 const (
@@ -19,11 +18,6 @@ const (
  HubModeSpeed = "speed"
 )
 
-type hostsQuery struct {
- Hosts []*info.Node `json:"hosts"`
- // Ignore other fields
-}
-
 // CheckValidMode checks if a given host selection/sync mode is valid or not.
 func CheckValidMode(mode string) error {
  switch mode {
@@ -35,31 +29,32 @@ func CheckValidMode(mode string) error {
 
 // QueryHub queries the BTFS-Hub to retrieve the latest list of hosts info
 // according to a certain mode.
-func QueryHub(node *core.IpfsNode, mode string) ([]*info.Node, error) {
- config, err := node.Repo.Config()
- if err != nil {
- return nil, err
- }
-
- hubUrl := config.Services.HubDomain
-
- params := "?id=" + node.Identity.Pretty()
+func QueryHub(ctx context.Context, node *core.IpfsNode, mode string) ([]*hubpb.Host, error) {
  switch mode {
  case HubModeScore:
  // Already the default on hub api
  default:
  return nil, fmt.Errorf(`Mode "%s" is not yet supported`, mode)
  }
 
- resp, err := http.Get(hubUrl + params)
+ config, err := node.Repo.Config()
  if err != nil {
- return nil, fmt.Errorf("Failed to query BTFS-Hub service: %v", err)
- }
-
- var hq hostsQuery
- if err := json.NewDecoder(resp.Body).Decode(&hq); err != nil {
  return nil, err
  }
+ var resp *hubpb.HostsResp
+ err = grpc.HubQueryClient(config.Services.HubDomain).WithContext(ctx, func(ctx context.Context,
+ client hubpb.HubQueryServiceClient) error {
+ resp, err = client.GetHosts(ctx, &hubpb.HostsReq{
+ Id: node.Identity.Pretty(),
+ })
+ if err != nil {
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("Failed to query BTFS-Hub service: %v", err)
+ }
 
- return hq.Hosts, nil
+ return resp.Hosts.Hosts, nil
 }