Skip to content

Commit

Permalink
cleanup httpfs client
Browse files Browse the repository at this point in the history
  • Loading branch information
barnex committed Jan 8, 2015
1 parent a38d0e1 commit 0220949
Show file tree
Hide file tree
Showing 3 changed files with 149 additions and 125 deletions.
131 changes: 131 additions & 0 deletions httpfs/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package httpfs

// client-side code

import (
"bytes"
"errors"
"io/ioutil"
"net/http"
"net/url"
"path"
"strings"
)

// SetWD sets a "working directory" for the client side,
// prefixed to all relative local paths passed to client functions (Mkdir, Touch, Remove, ...).
// dir may start with "http://", turning local relative client paths into remote paths.
// E.g.:
// http://path -> http://path
// path/file -> wd/path/file
// /path/file -> /path/file
func SetWD(dir string) {
if dir != "" && !strings.HasSuffix(dir, "/") {
dir = dir + "/"
}
wd = dir
}

// Creates the directory at specified URL (or local file),
// creating all needed parent directories as well.
func Mkdir(URL string) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpMkdir(URL)
} else {
return localMkdir(URL)
}
}

func Touch(URL string) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpTouch(URL)
} else {
return localTouch(URL)
}
}

func ReadDir(URL string) ([]string, error) {
URL = cleanup(URL)
if isRemote(URL) {
return httpLs(URL)
} else {
return localLs(URL)
}
}

func Remove(URL string) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpRemove(URL)
} else {
return localRemove(URL)
}
}

func Read(URL string) ([]byte, error) {
URL = cleanup(URL)
if isRemote(URL) {
return httpRead(URL)
} else {
return localRead(URL)
}
}

// Append p to the file given by URL,
// but first assure that the file had the expected size.
// Used to avoid accidental concurrent writes by two processes to the same file.
func AppendSize(URL string, p []byte, size int64) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpAppend(URL, p, size)
} else {
return localAppend(URL, p, size)
}
}

func Append(URL string, p []byte) error {
return AppendSize(URL, p, -1)
}

func Put(URL string, p []byte) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpPut(URL, p)
} else {
return localPut(URL, p)
}
}

func isRemote(URL string) bool {
return strings.HasPrefix(URL, "http://")
}

func cleanup(URL string) string {
if isRemote(URL) {
return URL
}
if !path.IsAbs(URL) {
return wd + URL
}
return URL
}

// TODO: query values
func do(a action, URL string, body []byte, query url.Values) (resp []byte, err error) {
u, err := url.Parse(URL)
u.Path = string(a) + path.Clean("/"+u.Path)
u.RawQuery = query.Encode()
response, errR := http.Post(u.String(), "data", bytes.NewReader(body))
if errR != nil {
return nil, mkErr(a, URL, errR)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.New("do " + u.String() + ":" + response.Status + ":" + readBody(response.Body))
}
resp, err = ioutil.ReadAll(response.Body)
err = mkErr(a, URL, err)
return
}
123 changes: 0 additions & 123 deletions httpfs/httpfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,13 @@ it is local. Hence, the same API is used for local and remote file access.
package httpfs

import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"path"
"strings"
"sync"
)

Expand All @@ -36,124 +31,6 @@ const (
FilePerm = 0666 // permissions for new files
)

// SetWD sets a "working directory" for the client side,
// prefixed to all relative local paths passed to client functions (Mkdir, Touch, Remove, ...).
// dir may start with "http://", turning local relative client paths into remote paths.
// E.g.:
// http://path -> http://path
// path/file -> wd/path/file
// /path/file -> /path/file
func SetWD(dir string) {
if dir != "" && !strings.HasSuffix(dir, "/") {
dir = dir + "/"
}
wd = dir
}

// Creates the directory at specified URL (or local file),
// creating all needed parent directories as well.
func Mkdir(URL string) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpMkdir(URL)
} else {
return localMkdir(URL)
}
}

func Touch(URL string) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpTouch(URL)
} else {
return localTouch(URL)
}
}

func ReadDir(URL string) ([]string, error) {
URL = cleanup(URL)
if isRemote(URL) {
return httpLs(URL)
} else {
return localLs(URL)
}
}

func Remove(URL string) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpRemove(URL)
} else {
return localRemove(URL)
}
}

func Read(URL string) ([]byte, error) {
URL = cleanup(URL)
if isRemote(URL) {
return httpRead(URL)
} else {
return localRead(URL)
}
}

// Append p to the file given by URL,
// but first assure that the file had the expected size.
// Used to avoid accidental concurrent writes by two processes to the same file.
func AppendSize(URL string, p []byte, size int64) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpAppend(URL, p, size)
} else {
return localAppend(URL, p, size)
}
}

func Append(URL string, p []byte) error {
return AppendSize(URL, p, -1)
}

func Put(URL string, p []byte) error {
URL = cleanup(URL)
if isRemote(URL) {
return httpPut(URL, p)
} else {
return localPut(URL, p)
}
}

func isRemote(URL string) bool {
return strings.HasPrefix(URL, "http://")
}

func cleanup(URL string) string {
if isRemote(URL) {
return URL
}
if !path.IsAbs(URL) {
return wd + URL
}
return URL
}

// TODO: query values
func do(a action, URL string, body []byte, query url.Values) (resp []byte, err error) {
u, err := url.Parse(URL)
u.Path = string(a) + path.Clean("/"+u.Path)
u.RawQuery = query.Encode()
response, errR := http.Post(u.String(), "data", bytes.NewReader(body))
if errR != nil {
return nil, mkErr(a, URL, errR)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.New("do " + u.String() + ":" + response.Status + ":" + readBody(response.Body))
}
resp, err = ioutil.ReadAll(response.Body)
err = mkErr(a, URL, err)
return
}

func readBody(r io.Reader) string {
b, err := ioutil.ReadAll(r)
if err != nil {
Expand Down
20 changes: 18 additions & 2 deletions httpfs/httpfs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import (
"testing"
)

// leaving this many files open is supposed to trigger os error.
const MANYFILES = 1025

// start local httpfs server, and use http://address/ as WD
func init() {
l, err := net.Listen("tcp", ":")
Expand All @@ -27,13 +30,26 @@ func init() {
}()
}

func TestMkdir(t *testing.T) {
func TestMkdirRemove(t *testing.T) {
_ = Remove("testdata")

mustPass(t, Mkdir("testdata"))
mustFail(t, Mkdir("testdata"))
mustFail(t, Mkdir("testdata"))

// test for closing files (internally)
for i := 0; i < MANYFILES; i++ {
mustPass(t, Remove("testdata"))
mustPass(t, Mkdir("testdata"))
}
}

func TestMkdir(t *testing.T) {
_ = Remove("testdata")

mustFail(t, Mkdir("testdata/bla/bla"))
mustPass(t, Mkdir("testdata/"))
mustPass(t, Mkdir("testdata/bla"))
mustPass(t, Mkdir("testdata/bla/bla"))
}

func mustPass(t *testing.T, err error) {
Expand Down

0 comments on commit 0220949

Please sign in to comment.