swarm/api: refactor and improve HTTP API (#3773)

This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).

cmd/swarm/list.go

@@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
 
 	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 	client := swarm.NewClient(bzzapi)
-	entries, err := client.ManifestFileList(manifest, prefix)
+	list, err := client.List(manifest, prefix)
 	if err != nil {
 		utils.Fatalf("Failed to generate file and directory list: %s", err)
 	}
@@ -52,7 +52,10 @@ func list(ctx *cli.Context) {
 	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
 	defer w.Flush()
 	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
-	for _, entry := range entries {
+	for _, prefix := range list.CommonPrefixes {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+	}
+	for _, entry := range list.Entries {
 		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
 	}
 }

cmd/swarm/manifest.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -42,7 +43,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 
 	if len(args) > 3 {
@@ -76,7 +77,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -106,7 +107,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -125,11 +126,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 		client           = swarm.NewClient(bzzapi)
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -163,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -173,9 +170,9 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := swarm.ManifestEntry{
-			Path:        path,
+		newEntry := api.ManifestEntry{
 			Hash:        hash,
+			Path:        path,
 			ContentType: ctype,
 		}
 		mroot.Entries = append(mroot.Entries, newEntry)
@@ -192,18 +189,10 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = swarm.NewClient(bzzapi)
-		newEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		newEntry         = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -237,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -250,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := swarm.ManifestEntry{
-					Path:        entry.Path,
+				myEntry := api.ManifestEntry{
 					Hash:        hash,
+					Path:        entry.Path,
 					ContentType: ctype,
 				}
 				newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -276,18 +265,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
-		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = swarm.NewClient(bzzapi)
-		entryToRemove = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		entryToRemove    = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -319,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -331,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)

cmd/swarm/upload.go

@@ -18,13 +18,15 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime"
+	"net/http"
 	"os"
 	"os/user"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -42,12 +44,10 @@ func upload(ctx *cli.Context) {
 		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
 		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
 		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client       = swarm.NewClient(bzzapi)
+		file         string
 	)
 
-	var client = swarm.NewClient(bzzapi)
-	var entry swarm.ManifestEntry
-	var file string
-
 	if len(args) != 1 {
 		if fromStdin {
 			tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -66,41 +66,47 @@ func upload(ctx *cli.Context) {
 			utils.Fatalf("Need filename as the first and only argument")
 		}
 	} else {
-		file = args[0]
+		file = expandPath(args[0])
+	}
+
+	if !wantManifest {
+		f, err := swarm.Open(file)
+		if err != nil {
+			utils.Fatalf("Error opening file: %s", err)
+		}
+		defer f.Close()
+		hash, err := client.UploadRaw(f, f.Size)
+		if err != nil {
+			utils.Fatalf("Upload failed: %s", err)
+		}
+		fmt.Println(hash)
+		return
 	}
 
-	fi, err := os.Stat(expandPath(file))
+	stat, err := os.Stat(file)
 	if err != nil {
-		utils.Fatalf("Failed to stat file: %v", err)
+		utils.Fatalf("Error opening file: %s", err)
 	}
-	if fi.IsDir() {
+	var hash string
+	if stat.IsDir() {
 		if !recursive {
 			utils.Fatalf("Argument is a directory and recursive upload is disabled")
 		}
-		if !wantManifest {
-			utils.Fatalf("Manifest is required for directory uploads")
+		hash, err = client.UploadDirectory(file, defaultPath, "")
+	} else {
+		if mimeType == "" {
+			mimeType = detectMimeType(file)
 		}
-		mhash, err := client.UploadDirectory(file, defaultPath)
+		f, err := swarm.Open(file)
 		if err != nil {
-			utils.Fatalf("Failed to upload directory: %v", err)
+			utils.Fatalf("Error opening file: %s", err)
 		}
-		fmt.Println(mhash)
-		return
+		defer f.Close()
+		f.ContentType = mimeType
+		hash, err = client.Upload(f, "")
 	}
-	entry, err = client.UploadFile(file, fi, mimeType)
 	if err != nil {
-		utils.Fatalf("Upload failed: %v", err)
-	}
-	mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
-	hash, err := client.UploadManifest(mroot)
-	if err != nil {
-		utils.Fatalf("Manifest upload failed: %v", err)
+		utils.Fatalf("Upload failed: %s", err)
 	}
 	fmt.Println(hash)
 }
@@ -128,3 +134,19 @@ func homeDir() string {
 	}
 	return ""
 }
+
+func detectMimeType(file string) string {
+	if ext := filepath.Ext(file); ext != "" {
+		return mime.TypeByExtension(ext)
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	buf := make([]byte, 512)
+	if n, _ := f.Read(buf); n > 0 {
+		return http.DetectContentType(buf)
+	}
+	return ""
+}

swarm/api/api.go

@@ -17,6 +17,7 @@
 package api
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -70,86 +71,50 @@ func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key stor
 type ErrResolve error
 
 // DNS Resolver
-func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
-	log.Trace(fmt.Sprintf("Resolving : %v", hostPort))
-	if hashMatcher.MatchString(hostPort) || self.dns == nil {
-		log.Trace(fmt.Sprintf("host is a contentHash: '%v'", hostPort))
-		return storage.Key(common.Hex2Bytes(hostPort)), nil
+func (self *Api) Resolve(uri *URI) (storage.Key, error) {
+	log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))
+	if hashMatcher.MatchString(uri.Addr) {
+		log.Trace(fmt.Sprintf("addr is a hash: %q", uri.Addr))
+		return storage.Key(common.Hex2Bytes(uri.Addr)), nil
 	}
-	if !nameresolver {
-		return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort)
+	if uri.Immutable() {
+		return nil, errors.New("refusing to resolve immutable address")
 	}
-	contentHash, err := self.dns.Resolve(hostPort)
-	if err != nil {
-		err = ErrResolve(err)
-		log.Warn(fmt.Sprintf("DNS error : %v", err))
-	}
-	log.Trace(fmt.Sprintf("host lookup: %v -> %v", hostPort, contentHash))
-	return contentHash[:], err
-}
-func Parse(uri string) (hostPort, path string) {
-	if uri == "" {
-		return
-	}
-	parts := slashes.Split(uri, 3)
-	var i int
-	if len(parts) == 0 {
-		return
+	if self.dns == nil {
+		return nil, fmt.Errorf("unable to resolve addr %q, resolver not configured", uri.Addr)
 	}
-	// beginning with slash is now optional
-	for len(parts[i]) == 0 {
-		i++
-	}
-	hostPort = parts[i]
-	for i < len(parts)-1 {
-		i++
-		if len(path) > 0 {
-			path = path + "/" + parts[i]
-		} else {
-			path = parts[i]
-		}
+	hash, err := self.dns.Resolve(uri.Addr)
+	if err != nil {
+		log.Warn(fmt.Sprintf("DNS error resolving addr %q: %s", uri.Addr, err))
+		return nil, ErrResolve(err)
 	}
-	log.Debug(fmt.Sprintf("host: '%s', path '%s' requested.", hostPort, path))
-	return
-}
-
-func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
-	hostPort, path = Parse(uri)
-	//resolving host and port
-	contentHash, err := self.Resolve(hostPort, nameresolver)
-	log.Debug(fmt.Sprintf("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path))
-	return contentHash[:], hostPort, path, err
+	log.Trace(fmt.Sprintf("addr lookup: %v -> %v", uri.Addr, hash))
+	return hash[:], nil
 }
 
 // Put provides singleton manifest creation on top of dpa store
-func (self *Api) Put(content, contentType string) (string, error) {
+func (self *Api) Put(content, contentType string) (storage.Key, error) {
 	r := strings.NewReader(content)
 	wg := &sync.WaitGroup{}
 	key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
 	r = strings.NewReader(manifest)
 	key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	wg.Wait()
-	return key.String(), nil
+	return key, nil
 }
 
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve path to content using dpa retrieve
 // it returns a section reader, mimeType, status and an error
-func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
-	key, _, path, err := self.parseAndResolve(uri, nameresolver)
-	if err != nil {
-		return nil, "", 500, fmt.Errorf("can't resolve: %v", err)
-	}
-
-	quitC := make(chan bool)
-	trie, err := loadManifest(self.dpa, key, quitC)
+func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
+	trie, err := loadManifest(self.dpa, key, nil)
 	if err != nil {
 		log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
 		return
@@ -173,32 +138,25 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
 	return
 }
 
-func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
-	root, _, path, err := self.parseAndResolve(uri, nameresolver)
-	if err != nil {
-		return "", fmt.Errorf("can't resolve: %v", err)
-	}
-
+func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
 	quitC := make(chan bool)
-	trie, err := loadManifest(self.dpa, root, quitC)
+	trie, err := loadManifest(self.dpa, key, quitC)
 	if err != nil {
-		return
+		return nil, err
 	}
-
 	if contentHash != "" {
-		entry := &manifestTrieEntry{
+		entry := newManifestTrieEntry(&ManifestEntry{
 			Path:        path,
-			Hash:        contentHash,
 			ContentType: contentType,
-		}
+		}, nil)
+		entry.Hash = contentHash
 		trie.addEntry(entry, quitC)
 	} else {
 		trie.deleteEntry(path, quitC)
 	}
 
-	err = trie.recalcAndStore()
-	if err != nil {
-		return
+	if err := trie.recalcAndStore(); err != nil {
+		return nil, err
 	}
-	return trie.hash.String(), nil
+	return trie.hash, nil
 }

swarm/api/api_test.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
@@ -81,8 +82,9 @@ func expResponse(content string, mimeType string, status int) *Response {
 }
 
 // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
-func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
-	reader, mimeType, status, err := api.Get(bzzhash, true)
+func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
+	key := storage.Key(common.Hex2Bytes(bzzhash))
+	reader, mimeType, status, err := api.Get(key, path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -107,11 +109,11 @@ func TestApiPut(t *testing.T) {
 		content := "hello"
 		exp := expResponse(content, "text/plain", 0)
 		// exp := expResponse([]byte(content), "text/plain", 0)
-		bzzhash, err := api.Put(content, exp.MimeType)
+		key, err := api.Put(content, exp.MimeType)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		resp := testGet(t, api, bzzhash)
+		resp := testGet(t, api, key.String(), "")
 		checkResponse(t, resp, exp)
 	})
 }

swarm/api/client/client.go

@@ -17,18 +17,23 @@
 package client
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime"
+	"mime/multipart"
 	"net/http"
+	"net/textproto"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 )
 
 var (
@@ -36,18 +41,6 @@ var (
 	DefaultClient  = NewClient(DefaultGateway)
 )
 
-// Manifest represents a swarm manifest.
-type Manifest struct {
-	Entries []ManifestEntry `json:"entries,omitempty"`
-}
-
-// ManifestEntry represents an entry in a swarm manifest.
-type ManifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
 func NewClient(gateway string) *Client {
 	return &Client{
 		Gateway: gateway,
@@ -59,160 +52,207 @@ type Client struct {
 	Gateway string
 }
 
-func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
+// UploadRaw uploads raw data to swarm and returns the resulting hash
+func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
+	if size <= 0 {
+		return "", errors.New("data size must be greater than zero")
 	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
+	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+	if err != nil {
+		return "", err
 	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) {
-	var mimetype string
-	hash, err := c.uploadFileContent(file, fi)
-	if mimetype_hint != "" {
-		mimetype = mimetype_hint
-		log.Info("Mime type set by override", "mime", mimetype)
-	} else {
-		ext := filepath.Ext(file)
-		log.Info("Ext", "ext", ext, "file", file)
-		if ext != "" {
-			mimetype = mime.TypeByExtension(filepath.Ext(fi.Name()))
-			log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file))
-		} else {
-			f, err := os.Open(file)
-			if err == nil {
-				first512 := make([]byte, 512)
-				fread, _ := f.ReadAt(first512, 0)
-				if fread > 0 {
-					mimetype = http.DetectContentType(first512[:fread])
-					log.Info("Mime type set by autodetection", "mime", mimetype)
-				}
-			}
-			f.Close()
-		}
-
+	req.ContentLength = size
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
 	}
-	m := ManifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	return m, err
-}
-
-func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
+	data, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return "", err
 	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
+	return string(data), nil
 }
 
-func (c *Client) UploadManifest(m Manifest) (string, error) {
-	jsm, err := json.Marshal(m)
+// DownloadRaw downloads raw data from swarm
+func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
+	uri := c.Gateway + "/bzzr:/" + hash
+	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		panic(err)
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		res.Body.Close()
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+	return res.Body, nil
 }
 
-func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+// File represents a file in a swarm manifest and is used for uploading and
+// downloading content to and from swarm
+type File struct {
+	io.ReadCloser
+	api.ManifestEntry
+}
+
+// Open opens a local file which can then be passed to client.Upload to upload
+// it to swarm
+func Open(path string) (*File, error) {
+	f, err := os.Open(path)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
+	stat, err := f.Stat()
 	if err != nil {
-		return "", err
+		f.Close()
+		return nil, err
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
+	return &File{
+		ReadCloser: f,
+		ManifestEntry: api.ManifestEntry{
+			ContentType: mime.TypeByExtension(filepath.Ext(path)),
+			Mode:        int64(stat.Mode()),
+			Size:        stat.Size(),
+			ModTime:     stat.ModTime(),
+		},
+	}, nil
+}
+
+// Upload uploads a file to swarm and either adds it to an existing manifest
+// (if the manifest argument is non-empty) or creates a new manifest containing
+// the file, returning the resulting manifest hash (the file will then be
+// available at bzz:/<hash>/<path>)
+func (c *Client) Upload(file *File, manifest string) (string, error) {
+	if file.Size <= 0 {
+		return "", errors.New("file size must be greater than zero")
 	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
+	return c.TarUpload(manifest, &FileUploader{file})
 }
 
-func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+// Download downloads a file with the given path from the swarm manifest with
+// the given hash (i.e. it gets bzz:/<hash>/<path>)
+func (c *Client) Download(hash, path string) (*File, error) {
+	uri := c.Gateway + "/bzz:/" + hash + "/" + path
+	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
+	if res.StatusCode != http.StatusOK {
+		res.Body.Close()
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	return &File{
+		ReadCloser: res.Body,
+		ManifestEntry: api.ManifestEntry{
+			ContentType: res.Header.Get("Content-Type"),
+			Size:        res.ContentLength,
+		},
+	}, nil
+}
+
+// UploadDirectory uploads a directory tree to swarm and either adds the files
+// to an existing manifest (if the manifest argument is non-empty) or creates a
+// new manifest, returning the resulting manifest hash (files from the
+// directory will then be available at bzz:/<hash>/path/to/file), with
+// the file specified in defaultPath being uploaded to the root of the manifest
+// (i.e. bzz:/<hash>/)
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
+	stat, err := os.Stat(dir)
 	if err != nil {
 		return "", err
+	} else if !stat.IsDir() {
+		return "", fmt.Errorf("not a directory: %s", dir)
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
+	return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
 }
 
-func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+// DownloadDirectory downloads the files contained in a swarm manifest under
+// the given path into a local directory (existing files will be overwritten)
+func (c *Client) DownloadDirectory(hash, path, destDir string) error {
+	stat, err := os.Stat(destDir)
+	if err != nil {
+		return err
+	} else if !stat.IsDir() {
+		return fmt.Errorf("not a directory: %s", destDir)
+	}
 
-	mroot := Manifest{}
-	req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+	uri := c.Gateway + "/bzz:/" + hash + "/" + path
+	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {
-		return mroot, err
+		return err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	req.Header.Set("Accept", "application/x-tar")
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return mroot, err
+		return err
 	}
-	defer resp.Body.Close()
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	tr := tar.NewReader(res.Body)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		// ignore the default path file
+		if hdr.Name == "" {
+			continue
+		}
 
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
+		dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
+		if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+			return err
+		}
+		var mode os.FileMode = 0644
+		if hdr.Mode > 0 {
+			mode = os.FileMode(hdr.Mode)
+		}
+		dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+		if err != nil {
+			return err
+		}
+		n, err := io.Copy(dst, tr)
+		dst.Close()
+		if err != nil {
+			return err
+		} else if n != hdr.Size {
+			return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
+		}
+	}
+}
 
+// UploadManifest uploads the given manifest to swarm
+func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
+	data, err := json.Marshal(m)
+	if err != nil {
+		return "", err
 	}
-	content, err := ioutil.ReadAll(resp.Body)
+	return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
+}
 
-	err = json.Unmarshal(content, &mroot)
+// DownloadManifest downloads a swarm manifest
+func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
+	res, err := c.DownloadRaw(hash)
 	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+		return nil, err
+	}
+	defer res.Close()
+	var manifest api.Manifest
+	if err := json.NewDecoder(res).Decode(&manifest); err != nil {
+		return nil, err
 	}
-	return mroot, err
+	return &manifest, nil
 }
 
-// ManifestFileList downloads the manifest with the given hash and generates a
-// list of files and directory prefixes which have the specified prefix.
+// List list files in a swarm manifest which have the given prefix, grouping
+// common prefixes using "/" as a delimiter.
 //
 // For example, if the manifest represents the following directory structure:
 //
@@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
 // - a prefix of ""      would return [dir1/, file1.txt, file2.txt]
 // - a prefix of "file"  would return [file1.txt, file2.txt]
 // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
-func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
-	manifest, err := c.DownloadManifest(hash)
+//
+// where entries ending with "/" are common prefixes.
+func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
+	res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
 	if err != nil {
 		return nil, err
 	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	var list api.ManifestList
+	if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
+		return nil, err
+	}
+	return &list, nil
+}
+
+// Uploader uploads files to swarm using a provided UploadFn
+type Uploader interface {
+	Upload(UploadFn) error
+}
+
+type UploaderFunc func(UploadFn) error
+
+func (u UploaderFunc) Upload(upload UploadFn) error {
+	return u(upload)
+}
+
+// DirectoryUploader uploads all files in a directory, optionally uploading
+// a file to the default path
+type DirectoryUploader struct {
+	Dir         string
+	DefaultPath string
+}
 
-	// handleFile handles a manifest entry which is a direct reference to a
-	// file (i.e. it is not a swarm manifest)
-	handleFile := func(entry ManifestEntry) {
-		// ignore the file if it doesn't have the specified prefix
-		if !strings.HasPrefix(entry.Path, prefix) {
-			return
+// Upload performs the upload of the directory and default path
+func (d *DirectoryUploader) Upload(upload UploadFn) error {
+	if d.DefaultPath != "" {
+		file, err := Open(d.DefaultPath)
+		if err != nil {
+			return err
 		}
-		// if the path after the prefix contains a directory separator,
-		// add a directory prefix to the entries, otherwise add the
-		// file
-		suffix := strings.TrimPrefix(entry.Path, prefix)
-		if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
-			entries = append(entries, ManifestEntry{
-				Path:        prefix + suffix[:sepIndex+1],
-				ContentType: "DIR",
-			})
-		} else {
-			if entry.Path == "" {
-				entry.Path = "/"
-			}
-			entries = append(entries, entry)
+		if err := upload(file); err != nil {
+			return err
 		}
 	}
-
-	// handleManifest handles a manifest entry which is a reference to
-	// another swarm manifest.
-	handleManifest := func(entry ManifestEntry) error {
-		// if the manifest's path is a prefix of the specified prefix
-		// then just recurse into the manifest by stripping its path
-		// from the prefix
-		if strings.HasPrefix(prefix, entry.Path) {
-			subPrefix := strings.TrimPrefix(prefix, entry.Path)
-			subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
-			if err != nil {
-				return err
-			}
-			// prefix the manifest's path to the sub entries and
-			// add them to the returned entries
-			for i, subEntry := range subEntries {
-				subEntry.Path = entry.Path + subEntry.Path
-				subEntries[i] = subEntry
-			}
-			entries = append(entries, subEntries...)
+	return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if f.IsDir() {
 			return nil
 		}
+		file, err := Open(path)
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(d.Dir, path)
+		if err != nil {
+			return err
+		}
+		file.Path = filepath.ToSlash(relPath)
+		return upload(file)
+	})
+}
 
-		// if the manifest's path has the specified prefix, then if the
-		// path after the prefix contains a directory separator, add a
-		// directory prefix to the entries, otherwise recurse into the
-		// manifest
-		if strings.HasPrefix(entry.Path, prefix) {
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			sepIndex := strings.Index(suffix, "/")
-			if sepIndex > -1 {
-				entries = append(entries, ManifestEntry{
-					Path:        prefix + suffix[:sepIndex+1],
-					ContentType: "DIR",
-				})
-				return nil
-			}
-			subEntries, err := c.ManifestFileList(entry.Hash, "")
-			if err != nil {
-				return err
-			}
-			// prefix the manifest's path to the sub entries and
-			// add them to the returned entries
-			for i, subEntry := range subEntries {
-				subEntry.Path = entry.Path + subEntry.Path
-				subEntries[i] = subEntry
-			}
-			entries = append(entries, subEntries...)
-			return nil
+// FileUploader uploads a single file
+type FileUploader struct {
+	File *File
+}
+
+// Upload performs the upload of the file
+func (f *FileUploader) Upload(upload UploadFn) error {
+	return upload(f.File)
+}
+
+// UploadFn is the type of function passed to an Uploader to perform the upload
+// of a single file (for example, a directory uploader would call a provided
+// UploadFn for each file in the directory tree)
+type UploadFn func(file *File) error
+
+// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
+// returning the resulting manifest hash
+func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
+	reqR, reqW := io.Pipe()
+	defer reqR.Close()
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/x-tar")
+
+	// use 'Expect: 100-continue' so we don't send the request body if
+	// the server refuses the request
+	req.Header.Set("Expect", "100-continue")
+
+	tw := tar.NewWriter(reqW)
+
+	// define an UploadFn which adds files to the tar stream
+	uploadFn := func(file *File) error {
+		hdr := &tar.Header{
+			Name:    file.Path,
+			Mode:    file.Mode,
+			Size:    file.Size,
+			ModTime: file.ModTime,
+			Xattrs: map[string]string{
+				"user.swarm.content-type": file.ContentType,
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
 		}
-		return nil
+		_, err = io.Copy(tw, file)
+		return err
 	}
 
-	for _, entry := range manifest.Entries {
-		if entry.ContentType == "application/bzz-manifest+json" {
-			if err := handleManifest(entry); err != nil {
-				return nil, err
-			}
-		} else {
-			handleFile(entry)
+	// run the upload in a goroutine so we can send the request headers and
+	// wait for a '100 Continue' response before sending the tar stream
+	go func() {
+		err := uploader.Upload(uploadFn)
+		if err == nil {
+			err = tw.Close()
 		}
+		reqW.CloseWithError(err)
+	}()
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+// MultipartUpload uses the given Uploader to upload files to swarm as a
+// multipart form, returning the resulting manifest hash
+func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
+	reqR, reqW := io.Pipe()
+	defer reqR.Close()
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	if err != nil {
+		return "", err
 	}
 
-	return
+	// use 'Expect: 100-continue' so we don't send the request body if
+	// the server refuses the request
+	req.Header.Set("Expect", "100-continue")
+
+	mw := multipart.NewWriter(reqW)
+	req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
+
+	// define an UploadFn which adds files to the multipart form
+	uploadFn := func(file *File) error {
+		hdr := make(textproto.MIMEHeader)
+		hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
+		hdr.Set("Content-Type", file.ContentType)
+		hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
+		w, err := mw.CreatePart(hdr)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(w, file)
+		return err
+	}
+
+	// run the upload in a goroutine so we can send the request headers and
+	// wait for a '100 Continue' response before sending the multipart form
+	go func() {
+		err := uploader.Upload(uploadFn)
+		if err == nil {
+			err = mw.Close()
+		}
+		reqW.CloseWithError(err)
+	}()
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
 }

swarm/api/client/client_test.go

@@ -17,6 +17,7 @@
 package client
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -24,52 +25,221 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
-func TestClientManifestFileList(t *testing.T) {
+// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
+func TestClientUploadDownloadRaw(t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
+	client := NewClient(srv.URL)
+
+	// upload some raw data
+	data := []byte("foo123")
+	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the same data
+	res, err := client.DownloadRaw(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Close()
+	gotData, err := ioutil.ReadAll(res)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(gotData, data) {
+		t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
+	}
+}
+
+// TestClientUploadDownloadFiles test uploading and downloading files to swarm
+// manifests
+func TestClientUploadDownloadFiles(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	client := NewClient(srv.URL)
+	upload := func(manifest, path string, data []byte) string {
+		file := &File{
+			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+			ManifestEntry: api.ManifestEntry{
+				Path:        path,
+				ContentType: "text/plain",
+				Size:        int64(len(data)),
+			},
+		}
+		hash, err := client.Upload(file, manifest)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return hash
+	}
+	checkDownload := func(manifest, path string, expected []byte) {
+		file, err := client.Download(manifest, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		if file.Size != int64(len(expected)) {
+			t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
+		}
+		if file.ContentType != file.ContentType {
+			t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType)
+		}
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
+		}
+	}
+
+	// upload a file to the root of a manifest
+	rootData := []byte("some-data")
+	rootHash := upload("", "", rootData)
+
+	// check we can download the root file
+	checkDownload(rootHash, "", rootData)
+
+	// upload another file to the same manifest
+	otherData := []byte("some-other-data")
+	newHash := upload(rootHash, "some/other/path", otherData)
+
+	// check we can download both files from the new manifest
+	checkDownload(newHash, "", rootData)
+	checkDownload(newHash, "some/other/path", otherData)
+
+	// replace the root file with different data
+	newHash = upload(newHash, "", otherData)
+
+	// check both files have the other data
+	checkDownload(newHash, "", otherData)
+	checkDownload(newHash, "some/other/path", otherData)
+}
+
+var testDirFiles = []string{
+	"file1.txt",
+	"file2.txt",
+	"dir1/file3.txt",
+	"dir1/file4.txt",
+	"dir2/file5.txt",
+	"dir2/dir3/file6.txt",
+	"dir2/dir4/file7.txt",
+	"dir2/dir4/file8.txt",
+}
+
+func newTestDirectory(t *testing.T) string {
 	dir, err := ioutil.TempDir("", "swarm-client-test")
 	if err != nil {
 		t.Fatal(err)
 	}
-	files := []string{
-		"file1.txt",
-		"file2.txt",
-		"dir1/file3.txt",
-		"dir1/file4.txt",
-		"dir2/file5.txt",
-		"dir2/dir3/file6.txt",
-		"dir2/dir4/file7.txt",
-		"dir2/dir4/file8.txt",
-	}
-	for _, file := range files {
+
+	for _, file := range testDirFiles {
 		path := filepath.Join(dir, file)
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error creating dir for %s: %s", path, err)
 		}
-		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+		if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error writing file %s: %s", path, err)
 		}
 	}
 
+	return dir
+}
+
+// TestClientUploadDownloadDirectory tests uploading and downloading a
+// directory of files to a swarm manifest
+func TestClientUploadDownloadDirectory(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	// upload the directory
 	client := NewClient(srv.URL)
+	defaultPath := filepath.Join(dir, testDirFiles[0])
+	hash, err := client.UploadDirectory(dir, defaultPath, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string, expected []byte) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected data to be %q, got %q", expected, data)
+		}
+	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file, []byte(file))
+	}
+
+	// check we can download the default path
+	checkDownloadFile("", []byte(testDirFiles[0]))
+
+	// check we can download the directory
+	tmp, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if err := client.DownloadDirectory(hash, "", tmp); err != nil {
+		t.Fatal(err)
+	}
+	for _, file := range testDirFiles {
+		data, err := ioutil.ReadFile(filepath.Join(tmp, file))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, []byte(file)) {
+			t.Fatalf("expected data to be %q, got %q", file, data)
+		}
+	}
+}
+
+// TestClientFileList tests listing files in a swarm manifest
+func TestClientFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
-	hash, err := client.UploadDirectory(dir, "")
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	client := NewClient(srv.URL)
+	hash, err := client.UploadDirectory(dir, "", "")
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
 
 	ls := func(prefix string) []string {
-		entries, err := client.ManifestFileList(hash, prefix)
+		list, err := client.List(hash, prefix)
 		if err != nil {
 			t.Fatal(err)
 		}
-		paths := make([]string, len(entries))
-		for i, entry := range entries {
-			paths[i] = entry.Path
+		paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
+		for _, prefix := range list.CommonPrefixes {
+			paths = append(paths, prefix)
+		}
+		for _, entry := range list.Entries {
+			paths = append(paths, entry.Path)
 		}
 		sort.Strings(paths)
 		return paths
@@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) {
 	for prefix, expected := range tests {
 		actual := ls(prefix)
 		if !reflect.DeepEqual(actual, expected) {
-			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+			t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
+		}
+	}
+}
+
+// TestClientMultipartUpload tests uploading files to swarm using a multipart
+// upload
+func TestClientMultipartUpload(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	// define an uploader which uploads testDirFiles with some data
+	data := []byte("some-data")
+	uploader := UploaderFunc(func(upload UploadFn) error {
+		for _, name := range testDirFiles {
+			file := &File{
+				ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+				ManifestEntry: api.ManifestEntry{
+					Path:        name,
+					ContentType: "text/plain",
+					Size:        int64(len(data)),
+				},
+			}
+			if err := upload(file); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// upload the files as a multipart upload
+	client := NewClient(srv.URL)
+	hash, err := client.MultipartUpload("", uploader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		gotData, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(gotData, data) {
+			t.Fatalf("expected data to be %q, got %q", data, gotData)
 		}
 	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file)
+	}
 }

swarm/api/filesystem.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"sync"
 
@@ -43,6 +44,8 @@ func NewFileSystem(api *Api) *FileSystem {
 // Upload replicates a local directory as a manifest file and uploads it
 // using dpa store
 // TODO: localpath should point to a manifest
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *FileSystem) Upload(lpath, index string) (string, error) {
 	var list []*manifestTrieEntry
 	localpath, err := filepath.Abs(filepath.Clean(lpath))
@@ -72,9 +75,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 				if path[:start] != localpath {
 					return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
 				}
-				entry := &manifestTrieEntry{
-					Path: filepath.ToSlash(path),
-				}
+				entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil)
 				list = append(list, entry)
 			}
 			return err
@@ -91,9 +92,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 		if localpath[:start] != dir {
 			return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
 		}
-		entry := &manifestTrieEntry{
-			Path: filepath.ToSlash(localpath),
-		}
+		entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil)
 		list = append(list, entry)
 	}
 
@@ -153,11 +152,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 		}
 		entry.Path = RegularSlashes(entry.Path[start:])
 		if entry.Path == index {
-			ientry := &manifestTrieEntry{
-				Path:        "",
-				Hash:        entry.Hash,
+			ientry := newManifestTrieEntry(&ManifestEntry{
 				ContentType: entry.ContentType,
-			}
+			}, nil)
+			ientry.Hash = entry.Hash
 			trie.addEntry(ientry, quitC)
 		}
 		trie.addEntry(entry, quitC)
@@ -174,6 +172,8 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 
 // Download replicates the manifest path structure on the local filesystem
 // under localpath
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *FileSystem) Download(bzzpath, localpath string) error {
 	lpath, err := filepath.Abs(filepath.Clean(localpath))
 	if err != nil {
@@ -185,10 +185,15 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
 	}
 
 	//resolving host and port
-	key, _, path, err := self.api.parseAndResolve(bzzpath, true)
+	uri, err := Parse(path.Join("bzz:/", bzzpath))
+	if err != nil {
+		return err
+	}
+	key, err := self.api.Resolve(uri)
 	if err != nil {
 		return err
 	}
+	path := uri.Path
 
 	if len(path) > 0 {
 		path += "/"

swarm/api/filesystem_test.go

@@ -23,6 +23,9 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
 var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
@@ -51,16 +54,17 @@ func TestApiDirUpload0(t *testing.T) {
 			t.Fatalf("unexpected error: %v", err)
 		}
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash+"/index.html")
+		resp := testGet(t, api, bzzhash, "index.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 
 		content = readPath(t, "testdata", "test0", "index.css")
-		resp = testGet(t, api, bzzhash+"/index.css")
+		resp = testGet(t, api, bzzhash, "index.css")
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, err = api.Get(bzzhash, true)
+		key := storage.Key(common.Hex2Bytes(bzzhash))
+		_, _, _, err = api.Get(key, "")
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
@@ -90,7 +94,8 @@ func TestApiDirUploadModify(t *testing.T) {
 			return
 		}
 
-		bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true)
+		key := storage.Key(common.Hex2Bytes(bzzhash))
+		key, err = api.Modify(key, "index.html", "", "")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -107,32 +112,33 @@ func TestApiDirUploadModify(t *testing.T) {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
+		key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
+		key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
+		bzzhash = key.String()
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash+"/index2.html")
+		resp := testGet(t, api, bzzhash, "index2.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 
-		resp = testGet(t, api, bzzhash+"/img/logo.png")
+		resp = testGet(t, api, bzzhash, "img/logo.png")
 		exp = expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 
 		content = readPath(t, "testdata", "test0", "index.css")
-		resp = testGet(t, api, bzzhash+"/index.css")
+		resp = testGet(t, api, bzzhash, "index.css")
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, err = api.Get(bzzhash, true)
+		_, _, _, err = api.Get(key, "")
 		if err == nil {
 			t.Errorf("expected error: %v", err)
 		}
@@ -149,7 +155,7 @@ func TestApiDirUploadWithRootFile(t *testing.T) {
 		}
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash)
+		resp := testGet(t, api, bzzhash, "")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 	})
@@ -165,7 +171,7 @@ func TestApiFileUpload(t *testing.T) {
 		}
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash+"/index.html")
+		resp := testGet(t, api, bzzhash, "index.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 	})
@@ -181,7 +187,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) {
 		}
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash)
+		resp := testGet(t, api, bzzhash, "")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 	})

swarm/api/http/roundtripper_test.go

@@ -18,14 +18,14 @@ package http
 
 import (
 	"io/ioutil"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 )
 
-const port = "3222"
-
 func TestRoundTripper(t *testing.T) {
 	serveMux := http.NewServeMux()
 	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -36,9 +36,12 @@ func TestRoundTripper(t *testing.T) {
 			http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
 		}
 	})
-	go http.ListenAndServe(":"+port, serveMux)
 
-	rt := &RoundTripper{Port: port}
+	srv := httptest.NewServer(serveMux)
+	defer srv.Close()
+
+	host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
+	rt := &RoundTripper{Host: host, Port: port}
 	trans := &http.Transport{}
 	trans.RegisterProtocol("bzz", rt)
 	client := &http.Client{Transport: trans}

swarm/api/http/server.go

@@ -20,13 +20,19 @@ A simple http server interface to Swarm
 package http
 
 import (
-	"bytes"
+	"archive/tar"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
 	"net/http"
-	"regexp"
+	"os"
+	"path"
+	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -36,26 +42,6 @@ import (
 	"github.com/rs/cors"
 )
 
-const (
-	rawType = "application/octet-stream"
-)
-
-var (
-	// accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw)
-	bzzPrefix       = regexp.MustCompile("^/+bzz[ir]?:/+")
-	trailingSlashes = regexp.MustCompile("/+$")
-	rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$")
-	// forever         = func() time.Time { return time.Unix(0, 0) }
-	forever = time.Now
-)
-
-type sequentialReader struct {
-	reader io.Reader
-	pos    int64
-	ahead  map[int64](chan bool)
-	lock   sync.Mutex
-}
-
 // ServerConfig is the basic configuration needed for the HTTP server and also
 // includes CORS settings.
 type ServerConfig struct {
@@ -94,242 +80,569 @@ type Server struct {
 	api *api.Api
 }
 
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	requestURL := r.URL
-	// This is wrong
-	//	if requestURL.Host == "" {
-	//		var err error
-	//		requestURL, err = url.Parse(r.Referer() + requestURL.String())
-	//		if err != nil {
-	//			http.Error(w, err.Error(), http.StatusBadRequest)
-	//			return
-	//		}
-	//	}
-	log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")))
-	uri := requestURL.Path
-	var raw, nameresolver bool
-	var proto string
-
-	// HTTP-based URL protocol handler
-	log.Debug(fmt.Sprintf("BZZ request URI: '%s'", uri))
-
-	path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string {
-		proto = p
-		return ""
-	})
+// Request wraps http.Request and also includes the parsed bzz URI
+type Request struct {
+	http.Request
+
+	uri *api.URI
+}
 
-	// protocol identification (ugly)
-	if proto == "" {
-		log.Error(fmt.Sprintf("[BZZ] Swarm: Protocol error in request `%s`.", uri))
-		http.Error(w, "Invalid request URL: need access protocol (bzz:/, bzzr:/, bzzi:/) as first element in path.", http.StatusBadRequest)
+// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// body in swarm and returns the resulting storage key as a text/plain response
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
+	if r.uri.Path != "" {
+		s.BadRequest(w, r, "raw POST request cannot contain a path")
 		return
 	}
-	if len(proto) > 4 {
-		raw = proto[1:5] == "bzzr"
-		nameresolver = proto[1:5] != "bzzi"
+
+	if r.Header.Get("Content-Length") == "" {
+		s.BadRequest(w, r, "missing Content-Length header in request")
+		return
 	}
 
-	log.Debug("", "msg", log.Lazy{Fn: func() string {
-		return fmt.Sprintf("[BZZ] Swarm: %s request over protocol %s '%s' received.", r.Method, proto, path)
-	}})
+	key, err := s.api.Store(r.Body, r.ContentLength, nil)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+	s.logDebug("content for %s stored", key.Log())
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, key)
+}
+
+// HandlePostFiles handles a POST request (or deprecated PUT request) to
+// bzz:/<hash>/<path> which contains either a single file or multiple files
+// (either a tar archive or multipart form), adds those files either to an
+// existing manifest or to a new manifest under <path> and returns the
+// resulting manifest hash as a text/plain response
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
+	contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	if err != nil {
+		s.BadRequest(w, r, err.Error())
+		return
+	}
 
-	switch {
-	case r.Method == "POST" || r.Method == "PUT":
-		if r.Header.Get("content-length") == "" {
-			http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
+	var key storage.Key
+	if r.uri.Addr != "" {
+		key, err = s.api.Resolve(r.uri)
+		if err != nil {
+			s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
 			return
 		}
-		key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
-		if err == nil {
-			log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
-		} else {
-			http.Error(w, err.Error(), http.StatusBadRequest)
+	} else {
+		key, err = s.api.NewManifest()
+		if err != nil {
+			s.Error(w, r, err)
 			return
 		}
-		if r.Method == "POST" {
-			if raw {
-				w.Header().Set("Content-Type", "text/plain")
-				http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key))))
-			} else {
-				http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest)
-				return
+	}
+
+	newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
+		switch contentType {
+
+		case "application/x-tar":
+			return s.handleTarUpload(r, mw)
+
+		case "multipart/form-data":
+			return s.handleMultipartUpload(r, params["boundary"], mw)
+
+		default:
+			return s.handleDirectUpload(r, mw)
+		}
+	})
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, newKey)
+}
+
+func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
+	tr := tar.NewReader(req.Body)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return fmt.Errorf("error reading tar stream: %s", err)
+		}
+
+		// only store regular files
+		if !hdr.FileInfo().Mode().IsRegular() {
+			continue
+		}
+
+		// add the entry under the path from the request
+		path := path.Join(req.uri.Path, hdr.Name)
+		entry := &api.ManifestEntry{
+			Path:        path,
+			ContentType: hdr.Xattrs["user.swarm.content-type"],
+			Mode:        hdr.Mode,
+			Size:        hdr.Size,
+			ModTime:     hdr.ModTime,
+		}
+		s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
+		contentKey, err := mw.AddEntry(tr, entry)
+		if err != nil {
+			return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
+		}
+		s.logDebug("content for %s stored", contentKey.Log())
+	}
+}
+
+func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
+	mr := multipart.NewReader(req.Body, boundary)
+	for {
+		part, err := mr.NextPart()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return fmt.Errorf("error reading multipart form: %s", err)
+		}
+
+		var size int64
+		var reader io.Reader = part
+		if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
+			size, err = strconv.ParseInt(contentLength, 10, 64)
+			if err != nil {
+				return fmt.Errorf("error parsing multipart content length: %s", err)
 			}
+			reader = part
 		} else {
-			// PUT
-			if raw {
-				http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest)
-				return
-			} else {
-				path = api.RegularSlashes(path)
-				mime := r.Header.Get("Content-Type")
-				// TODO proper root hash separation
-				log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
-				newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
-				if err == nil {
-					log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
-					w.Header().Set("Content-Type", "text/plain")
-					http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
-				} else {
-					http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest)
-					return
-				}
+			// copy the part to a tmp file to get its size
+			tmp, err := ioutil.TempFile("", "swarm-multipart")
+			if err != nil {
+				return err
 			}
-		}
-	case r.Method == "DELETE":
-		if raw {
-			http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest)
-			return
-		} else {
-			path = api.RegularSlashes(path)
-			log.Debug(fmt.Sprintf("Delete '%s'.", path))
-			newKey, err := s.api.Modify(path, "", "", nameresolver)
-			if err == nil {
-				log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
-				w.Header().Set("Content-Type", "text/plain")
-				http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
-			} else {
-				http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest)
-				return
+			defer os.Remove(tmp.Name())
+			defer tmp.Close()
+			size, err = io.Copy(tmp, part)
+			if err != nil {
+				return fmt.Errorf("error copying multipart content: %s", err)
+			}
+			if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
+				return fmt.Errorf("error copying multipart content: %s", err)
 			}
+			reader = tmp
+		}
+
+		// add the entry under the path from the request
+		name := part.FileName()
+		if name == "" {
+			name = part.FormName()
+		}
+		path := path.Join(req.uri.Path, name)
+		entry := &api.ManifestEntry{
+			Path:        path,
+			ContentType: part.Header.Get("Content-Type"),
+			Size:        size,
+			ModTime:     time.Now(),
 		}
-	case r.Method == "GET" || r.Method == "HEAD":
-		path = trailingSlashes.ReplaceAllString(path, "")
-		if path == "" {
-			http.Error(w, "Empty path not allowed", http.StatusBadRequest)
+		s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
+		contentKey, err := mw.AddEntry(reader, entry)
+		if err != nil {
+			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
+		}
+		s.logDebug("content for %s stored", contentKey.Log())
+	}
+}
+
+func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
+	key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
+		Path:        req.uri.Path,
+		ContentType: req.Header.Get("Content-Type"),
+		Mode:        0644,
+		Size:        req.ContentLength,
+		ModTime:     time.Now(),
+	})
+	if err != nil {
+		return err
+	}
+	s.logDebug("content for %s stored", key.Log())
+	return nil
+}
+
+// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
+// <path> from <manifest> and returns the resulting manifest hash as a
+// text/plain response
+func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
+		s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
+		return mw.RemoveEntry(r.uri.Path)
+	})
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, newKey)
+}
+
+// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// the raw content stored at the given storage key
+func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	// if path is set, interpret <key> as a manifest and return the
+	// raw entry at the given path
+	if r.uri.Path != "" {
+		walker, err := s.api.NewManifestWalker(key, nil)
+		if err != nil {
+			s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
 			return
 		}
-		if raw {
-			var reader storage.LazySectionReader
-			parsedurl, _ := api.Parse(path)
-
-			if parsedurl == path {
-				key, err := s.api.Resolve(parsedurl, nameresolver)
-				if err != nil {
-					log.Error(fmt.Sprintf("%v", err))
-					http.Error(w, err.Error(), http.StatusBadRequest)
-					return
-				}
-				reader = s.api.Retrieve(key)
-			} else {
-				var status int
-				readertmp, _, status, err := s.api.Get(path, nameresolver)
-				if err != nil {
-					http.Error(w, err.Error(), status)
-					return
-				}
-				reader = readertmp
+		var entry *api.ManifestEntry
+		walker.Walk(func(e *api.ManifestEntry) error {
+			// if the entry matches the path, set entry and stop
+			// the walk
+			if e.Path == r.uri.Path {
+				entry = e
+				// return an error to cancel the walk
+				return errors.New("found")
 			}
 
-			// retrieving content
-
-			quitC := make(chan bool)
-			size, err := reader.Size(quitC)
-			if err != nil {
-				log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
-				//An error on call to Size means we don't have the root chunk
-				http.Error(w, err.Error(), http.StatusNotFound)
-				return
+			// ignore non-manifest files
+			if e.ContentType != api.ManifestType {
+				return nil
 			}
-			log.Debug(fmt.Sprintf("Reading %d bytes.", size))
 
-			// setting mime type
-			qv := requestURL.Query()
-			mimeType := qv.Get("content_type")
-			if mimeType == "" {
-				mimeType = rawType
+			// if the manifest's path is a prefix of the
+			// requested path, recurse into it by returning
+			// nil and continuing the walk
+			if strings.HasPrefix(r.uri.Path, e.Path) {
+				return nil
 			}
 
-			w.Header().Set("Content-Type", mimeType)
-			http.ServeContent(w, r, uri, forever(), reader)
-			log.Debug(fmt.Sprintf("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType))
+			return api.SkipManifest
+		})
+		if entry == nil {
+			http.NotFound(w, &r.Request)
+			return
+		}
+		key = storage.Key(common.Hex2Bytes(entry.Hash))
+	}
 
-			// retrieve path via manifest
-		} else {
-			log.Debug(fmt.Sprintf("Structured GET request '%s' received.", uri))
-			// add trailing slash, if missing
-			if rootDocumentUri.MatchString(uri) {
-				http.Redirect(w, r, path+"/", http.StatusFound)
-				return
+	// check the root chunk exists by retrieving the file's size
+	reader := s.api.Retrieve(key)
+	if _, err := reader.Size(nil); err != nil {
+		s.logDebug("key not found %s: %s", key, err)
+		http.NotFound(w, &r.Request)
+		return
+	}
+
+	// allow the request to overwrite the content type using a query
+	// parameter
+	contentType := "application/octet-stream"
+	if typ := r.URL.Query().Get("content_type"); typ != "" {
+		contentType = typ
+	}
+	w.Header().Set("Content-Type", contentType)
+
+	http.ServeContent(w, &r.Request, "", time.Now(), reader)
+}
+
+// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
+// header of "application/x-tar" and returns a tar stream of all files
+// contained in the manifest
+func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
+	if r.uri.Path != "" {
+		s.BadRequest(w, r, "files request cannot contain a path")
+		return
+	}
+
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	walker, err := s.api.NewManifestWalker(key, nil)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+	w.Header().Set("Content-Type", "application/x-tar")
+	w.WriteHeader(http.StatusOK)
+
+	err = walker.Walk(func(entry *api.ManifestEntry) error {
+		// ignore manifests (walk will recurse into them)
+		if entry.ContentType == api.ManifestType {
+			return nil
+		}
+
+		// retrieve the entry's key and size
+		reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
+		size, err := reader.Size(nil)
+		if err != nil {
+			return err
+		}
+
+		// write a tar header for the entry
+		hdr := &tar.Header{
+			Name:    entry.Path,
+			Mode:    entry.Mode,
+			Size:    size,
+			ModTime: entry.ModTime,
+			Xattrs: map[string]string{
+				"user.swarm.content-type": entry.ContentType,
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+
+		// copy the file into the tar stream
+		n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
+		if err != nil {
+			return err
+		} else if n != size {
+			return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
+		}
+
+		return nil
+	})
+	if err != nil {
+		s.logError("error generating tar stream: %s", err)
+	}
+}
+
+// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
+// the "list" query parameter set to "true" and returns a list of all files
+// contained in <manifest> under <path> grouped into common prefixes using
+// "/" as a delimiter
+func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
+	// ensure the root path has a trailing slash so that relative URLs work
+	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+		http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+		return
+	}
+
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	walker, err := s.api.NewManifestWalker(key, nil)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+
+	var list api.ManifestList
+	prefix := r.uri.Path
+	err = walker.Walk(func(entry *api.ManifestEntry) error {
+		// handle non-manifest files
+		if entry.ContentType != api.ManifestType {
+			// ignore the file if it doesn't have the specified prefix
+			if !strings.HasPrefix(entry.Path, prefix) {
+				return nil
 			}
-			reader, mimeType, status, err := s.api.Get(path, nameresolver)
-			if err != nil {
-				if _, ok := err.(api.ErrResolve); ok {
-					log.Debug(fmt.Sprintf("%v", err))
-					status = http.StatusBadRequest
-				} else {
-					log.Debug(fmt.Sprintf("error retrieving '%s': %v", uri, err))
-					status = http.StatusNotFound
-				}
-				http.Error(w, err.Error(), status)
-				return
+
+			// if the path after the prefix contains a slash, add a
+			// common prefix to the list, otherwise add the entry
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return nil
 			}
-			// set mime type and status headers
-			w.Header().Set("Content-Type", mimeType)
-			if status > 0 {
-				w.WriteHeader(status)
-			} else {
-				status = 200
+			if entry.Path == "" {
+				entry.Path = "/"
 			}
-			quitC := make(chan bool)
-			size, err := reader.Size(quitC)
-			if err != nil {
-				log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
-				//An error on call to Size means we don't have the root chunk
-				http.Error(w, err.Error(), http.StatusNotFound)
-				return
+			list.Entries = append(list.Entries, entry)
+			return nil
+		}
+
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by returning nil and
+		// continuing the walk
+		if strings.HasPrefix(prefix, entry.Path) {
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a slash, add a common prefix
+		// to the list and skip the manifest, otherwise recurse into
+		// the manifest by returning nil and continuing the walk
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return api.SkipManifest
 			}
-			log.Debug(fmt.Sprintf("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status))
+			return nil
+		}
 
-			http.ServeContent(w, r, path, forever(), reader)
+		// the manifest neither has the prefix or needs recursing in to
+		// so just skip it
+		return api.SkipManifest
+	})
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
 
+	// if the client wants HTML (e.g. a browser) then render the list as a
+	// HTML index with relative URLs
+	if strings.Contains(r.Header.Get("Accept"), "text/html") {
+		w.Header().Set("Content-Type", "text/html")
+		err := htmlListTemplate.Execute(w, &htmlListData{
+			URI:  r.uri,
+			List: &list,
+		})
+		if err != nil {
+			s.logError("error rendering list HTML: %s", err)
 		}
-	default:
-		http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+		return
 	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(&list)
 }
 
-func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) {
-	self.lock.Lock()
-	// assert self.pos <= off
-	if self.pos > off {
-		log.Error(fmt.Sprintf("non-sequential read attempted from sequentialReader; %d > %d", self.pos, off))
-		panic("Non-sequential read attempt")
-	}
-	if self.pos != off {
-		log.Debug(fmt.Sprintf("deferred read in POST at position %d, offset %d.", self.pos, off))
-		wait := make(chan bool)
-		self.ahead[off] = wait
-		self.lock.Unlock()
-		if <-wait {
-			// failed read behind
-			n = 0
-			err = io.ErrUnexpectedEOF
+// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
+// with the content of the file at <path> from the given <manifest>
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	reader, contentType, _, err := s.api.Get(key, r.uri.Path)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+
+	// check the root chunk exists by retrieving the file's size
+	if _, err := reader.Size(nil); err != nil {
+		s.logDebug("file not found %s: %s", r.uri, err)
+		http.NotFound(w, &r.Request)
+		return
+	}
+
+	w.Header().Set("Content-Type", contentType)
+
+	http.ServeContent(w, &r.Request, "", time.Now(), reader)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
+
+	uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+	if err != nil {
+		s.logError("Invalid URI %q: %s", r.URL.Path, err)
+		http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest)
+		return
+	}
+	s.logDebug("%s request received for %s", r.Method, uri)
+
+	req := &Request{Request: *r, uri: uri}
+	switch r.Method {
+	case "POST":
+		if uri.Raw() {
+			s.HandlePostRaw(w, req)
+		} else {
+			s.HandlePostFiles(w, req)
+		}
+
+	case "PUT":
+		// DEPRECATED:
+		//   clients should send a POST request (the request creates a
+		//   new manifest leaving the existing one intact, so it isn't
+		//   strictly a traditional PUT request which replaces content
+		//   at a URI, and POST is more ubiquitous)
+		if uri.Raw() {
+			http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
+		} else {
+			s.HandlePostFiles(w, req)
 		}
-		self.lock.Lock()
-	}
-	localPos := 0
-	for localPos < len(target) {
-		n, err = self.reader.Read(target[localPos:])
-		localPos += n
-		log.Debug(fmt.Sprintf("Read %d bytes into buffer size %d from POST, error %v.", n, len(target), err))
-		if err != nil {
-			log.Debug(fmt.Sprintf("POST stream's reading terminated with %v.", err))
-			for i := range self.ahead {
-				self.ahead[i] <- true
-				delete(self.ahead, i)
-			}
-			self.lock.Unlock()
-			return localPos, err
+
+	case "DELETE":
+		if uri.Raw() {
+			http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
+			return
+		}
+		s.HandleDelete(w, req)
+
+	case "GET":
+		if uri.Raw() {
+			s.HandleGetRaw(w, req)
+			return
+		}
+
+		if r.Header.Get("Accept") == "application/x-tar" {
+			s.HandleGetFiles(w, req)
+			return
+		}
+
+		if r.URL.Query().Get("list") == "true" {
+			s.HandleGetList(w, req)
+			return
 		}
-		self.pos += int64(n)
+
+		s.HandleGetFile(w, req)
+
+	default:
+		http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+
+	}
+}
+
+func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) {
+	mw, err := s.api.NewManifestWriter(key, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := update(mw); err != nil {
+		return nil, err
 	}
-	wait := self.ahead[self.pos]
-	if wait != nil {
-		log.Debug(fmt.Sprintf("deferred read in POST at position %d triggered.", self.pos))
-		delete(self.ahead, self.pos)
-		close(wait)
+
+	key, err = mw.Store()
+	if err != nil {
+		return nil, err
 	}
-	self.lock.Unlock()
-	return localPos, err
+	s.logDebug("generated manifest %s", key)
+	return key, nil
+}
+
+func (s *Server) logDebug(format string, v ...interface{}) {
+	log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
+}
+
+func (s *Server) logError(format string, v ...interface{}) {
+	log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
+}
+
+func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
+	s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason)
+	http.Error(w, reason, http.StatusBadRequest)
+}
+
+func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
+	s.logError("error serving %s %s: %s", r.Method, r.uri, err)
+	http.Error(w, err.Error(), http.StatusInternalServerError)
 }

swarm/api/http/server_test.go

@@ -40,8 +40,8 @@ func TestBzzrGetPath(t *testing.T) {
 
 	testrequests := make(map[string]int)
 	testrequests["/"] = 0
-	testrequests["/a"] = 1
-	testrequests["/a/b"] = 2
+	testrequests["/a/"] = 1
+	testrequests["/a/b/"] = 2
 	testrequests["/x"] = 0
 	testrequests[""] = 0
 

swarm/api/http/templates.go

@@ -0,0 +1,71 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"html/template"
+	"path"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+)
+
+type htmlListData struct {
+	URI  *api.URI
+	List *api.ManifestList
+}
+
+var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(`
+<!DOCTYPE html>
+<html>
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Swarm index of {{ .URI }}</title>
+</head>
+
+<body>
+  <h1>Swarm index of {{ .URI }}</h1>
+  <hr>
+  <table>
+    <thead>
+      <tr>
+	<th>Path</th>
+	<th>Type</th>
+	<th>Size</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      {{ range .List.CommonPrefixes }}
+	<tr>
+	  <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+	  <td>DIR</td>
+	  <td>-</td>
+	</tr>
+      {{ end }}
+
+      {{ range .List.Entries }}
+	<tr>
+	  <td><a href="{{ basename .Path }}">{{ basename .Path }}</a></td>
+	  <td>{{ .ContentType }}</td>
+	  <td>{{ .Size }}</td>
+	</tr>
+      {{ end }}
+  </table>
+  <hr>
+</body>
+`[1:]))

swarm/api/manifest.go

@@ -19,8 +19,11 @@ package api
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -28,25 +31,152 @@ import (
 )
 
 const (
-	manifestType = "application/bzz-manifest+json"
+	ManifestType = "application/bzz-manifest+json"
 )
 
+// Manifest represents a swarm manifest
+type Manifest struct {
+	Entries []ManifestEntry `json:"entries,omitempty"`
+}
+
+// ManifestEntry represents an entry in a swarm manifest
+type ManifestEntry struct {
+	Hash        string    `json:"hash,omitempty"`
+	Path        string    `json:"path,omitempty"`
+	ContentType string    `json:"contentType,omitempty"`
+	Mode        int64     `json:"mode,omitempty"`
+	Size        int64     `json:"size,omitempty"`
+	ModTime     time.Time `json:"mod_time,omitempty"`
+	Status      int       `json:"status,omitempty"`
+}
+
+// ManifestList represents the result of listing files in a manifest
+type ManifestList struct {
+	CommonPrefixes []string         `json:"common_prefixes,omitempty"`
+	Entries        []*ManifestEntry `json:"entries,omitempty"`
+}
+
+// NewManifest creates and stores a new, empty manifest
+func (a *Api) NewManifest() (storage.Key, error) {
+	var manifest Manifest
+	data, err := json.Marshal(&manifest)
+	if err != nil {
+		return nil, err
+	}
+	return a.Store(bytes.NewReader(data), int64(len(data)), nil)
+}
+
+// ManifestWriter is used to add and remove entries from an underlying manifest
+type ManifestWriter struct {
+	api   *Api
+	trie  *manifestTrie
+	quitC chan bool
+}
+
+func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) {
+	trie, err := loadManifest(a.dpa, key, quitC)
+	if err != nil {
+		return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
+	}
+	return &ManifestWriter{a, trie, quitC}, nil
+}
+
+// AddEntry stores the given data and adds the resulting key to the manifest
+func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
+	key, err := m.api.Store(data, e.Size, nil)
+	if err != nil {
+		return nil, err
+	}
+	entry := newManifestTrieEntry(e, nil)
+	entry.Hash = key.String()
+	m.trie.addEntry(entry, m.quitC)
+	return key, nil
+}
+
+// RemoveEntry removes the given path from the manifest
+func (m *ManifestWriter) RemoveEntry(path string) error {
+	m.trie.deleteEntry(path, m.quitC)
+	return nil
+}
+
+// Store stores the manifest, returning the resulting storage key
+func (m *ManifestWriter) Store() (storage.Key, error) {
+	return m.trie.hash, m.trie.recalcAndStore()
+}
+
+// ManifestWalker is used to recursively walk the entries in the manifest and
+// all of its submanifests
+type ManifestWalker struct {
+	api   *Api
+	trie  *manifestTrie
+	quitC chan bool
+}
+
+func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) {
+	trie, err := loadManifest(a.dpa, key, quitC)
+	if err != nil {
+		return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
+	}
+	return &ManifestWalker{a, trie, quitC}, nil
+}
+
+// SkipManifest is used as a return value from WalkFn to indicate that the
+// manifest should be skipped
+var SkipManifest = errors.New("skip this manifest")
+
+// WalkFn is the type of function called for each entry visited by a recursive
+// manifest walk
+type WalkFn func(entry *ManifestEntry) error
+
+// Walk recursively walks the manifest calling walkFn for each entry in the
+// manifest, including submanifests
+func (m *ManifestWalker) Walk(walkFn WalkFn) error {
+	return m.walk(m.trie, "", walkFn)
+}
+
+func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
+	for _, entry := range trie.entries {
+		if entry == nil {
+			continue
+		}
+		entry.Path = prefix + entry.Path
+		err := walkFn(&entry.ManifestEntry)
+		if err != nil {
+			if entry.ContentType == ManifestType && err == SkipManifest {
+				continue
+			}
+			return err
+		}
+		if entry.ContentType != ManifestType {
+			continue
+		}
+		if err := trie.loadSubTrie(entry, nil); err != nil {
+			return err
+		}
+		if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 type manifestTrie struct {
 	dpa     *storage.DPA
 	entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
 	hash    storage.Key             // if hash != nil, it is stored
 }
 
-type manifestJSON struct {
-	Entries []*manifestTrieEntry `json:"entries"`
+func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
+	return &manifestTrieEntry{
+		ManifestEntry: *entry,
+		subtrie:       subtrie,
+	}
 }
 
 type manifestTrieEntry struct {
-	Path        string `json:"path"`
-	Hash        string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
-	ContentType string `json:"contentType"`
-	Status      int    `json:"status"`
-	subtrie     *manifestTrie
+	ManifestEntry
+
+	subtrie *manifestTrie
 }
 
 func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
@@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
 	}
 
 	log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
-	man := manifestJSON{}
+	var man struct {
+		Entries []*manifestTrieEntry `json:"entries"`
+	}
 	err = json.Unmarshal(manifestData, &man)
 	if err != nil {
 		err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
@@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 		cpl++
 	}
 
-	if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
+	if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
 		if self.loadSubTrie(oldentry, quitC) != nil {
 			return
 		}
@@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 	subtrie.addEntry(entry, quitC)
 	subtrie.addEntry(oldentry, quitC)
 
-	self.entries[b] = &manifestTrieEntry{
+	self.entries[b] = newManifestTrieEntry(&ManifestEntry{
 		Path:        commonPrefix,
-		Hash:        "",
-		ContentType: manifestType,
-		subtrie:     subtrie,
-	}
+		ContentType: ManifestType,
+	}, subtrie)
 }
 
 func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
 	}
 
 	epl := len(entry.Path)
-	if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
+	if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
 		if self.loadSubTrie(entry, quitC) != nil {
 			return
 		}
@@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error {
 	var buffer bytes.Buffer
 	buffer.WriteString(`{"entries":[`)
 
-	list := &manifestJSON{}
+	list := &Manifest{}
 	for _, entry := range self.entries {
 		if entry != nil {
 			if entry.Hash == "" { // TODO: paralellize
@@ -208,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error {
 				}
 				entry.Hash = entry.subtrie.hash.String()
 			}
-			list.Entries = append(list.Entries, entry)
+			list.Entries = append(list.Entries, entry.ManifestEntry)
 		}
 	}
 
@@ -254,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
 		entry := self.entries[i]
 		if entry != nil {
 			epl := len(entry.Path)
-			if entry.ContentType == manifestType {
+			if entry.ContentType == ManifestType {
 				l := plen
 				if epl < l {
 					l = epl
@@ -300,7 +430,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
 	log.Trace(fmt.Sprintf("path = %v  entry.Path = %v  epl = %v", path, entry.Path, epl))
 	if (len(path) >= epl) && (path[:epl] == entry.Path) {
 		log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
-		if entry.ContentType == manifestType {
+		if entry.ContentType == ManifestType {
 			err := self.loadSubTrie(entry, quitC)
 			if err != nil {
 				return nil, 0

swarm/api/storage.go

@@ -16,6 +16,8 @@
 
 package api
 
+import "path"
+
 type Response struct {
 	MimeType string
 	Status   int
@@ -25,6 +27,8 @@ type Response struct {
 }
 
 // implements a service
+//
+// DEPRECATED: Use the HTTP API instead
 type Storage struct {
 	api *Api
 }
@@ -35,8 +39,14 @@ func NewStorage(api *Api) *Storage {
 
 // Put uploads the content to the swarm with a simple manifest speficying
 // its content type
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *Storage) Put(content, contentType string) (string, error) {
-	return self.api.Put(content, contentType)
+	key, err := self.api.Put(content, contentType)
+	if err != nil {
+		return "", err
+	}
+	return key.String(), err
 }
 
 // Get retrieves the content from bzzpath and reads the response in full
@@ -45,8 +55,18 @@ func (self *Storage) Put(content, contentType string) (string, error) {
 // NOTE: if error is non-nil, sResponse may still have partial content
 // the actual size of which is given in len(resp.Content), while the expected
 // size is resp.Size
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *Storage) Get(bzzpath string) (*Response, error) {
-	reader, mimeType, status, err := self.api.Get(bzzpath, true)
+	uri, err := Parse(path.Join("bzz:/", bzzpath))
+	if err != nil {
+		return nil, err
+	}
+	key, err := self.api.Resolve(uri)
+	if err != nil {
+		return nil, err
+	}
+	reader, mimeType, status, err := self.api.Get(key, uri.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -65,6 +85,20 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
 
 // Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
 // and merge on  to it. creating an entry w conentType (mime)
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
-	return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
+	uri, err := Parse("bzz:/" + rootHash)
+	if err != nil {
+		return "", err
+	}
+	key, err := self.api.Resolve(uri)
+	if err != nil {
+		return "", err
+	}
+	key, err = self.api.Modify(key, path, contentHash, contentType)
+	if err != nil {
+		return "", err
+	}
+	return key.String(), nil
 }

swarm/api/storage_test.go

@@ -36,7 +36,7 @@ func TestStoragePutGet(t *testing.T) {
 			t.Fatalf("unexpected error: %v", err)
 		}
 		// to check put against the Api#Get
-		resp0 := testGet(t, api.api, bzzhash)
+		resp0 := testGet(t, api.api, bzzhash, "")
 		checkResponse(t, resp0, exp)
 
 		// check storage#Get

swarm/api/swarmfs_unix.go

@@ -91,11 +91,16 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
 		return nil, fmt.Errorf("%s is already mounted", cleanedMountPoint)
 	}
 
-	key, _, path, err := self.swarmApi.parseAndResolve(mhash, true)
+	uri, err := Parse("bzz:/" + mhash)
 	if err != nil {
-		return nil, fmt.Errorf("can't resolve %q: %v", mhash, err)
+		return nil, err
+	}
+	key, err := self.swarmApi.Resolve(uri)
+	if err != nil {
+		return nil, err
 	}
 
+	path := uri.Path
 	if len(path) > 0 {
 		path += "/"
 	}

swarm/api/uri.go

@@ -0,0 +1,96 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// URI is a reference to content stored in swarm.
+type URI struct {
+	// Scheme has one of the following values:
+	//
+	// * bzz  - an entry in a swarm manifest
+	// * bzzr - raw swarm content
+	// * bzzi - immutable URI of an entry in a swarm manifest
+	//          (address is not resolved)
+	Scheme string
+
+	// Addr is either a hexadecimal storage key or it an address which
+	// resolves to a storage key
+	Addr string
+
+	// Path is the path to the content within a swarm manifest
+	Path string
+}
+
+// Parse parses rawuri into a URI struct, where rawuri is expected to have one
+// of the following formats:
+//
+// * <scheme>:/
+// * <scheme>:/<addr>
+// * <scheme>:/<addr>/<path>
+// * <scheme>://
+// * <scheme>://<addr>
+// * <scheme>://<addr>/<path>
+//
+// with scheme one of bzz, bzzr or bzzi
+func Parse(rawuri string) (*URI, error) {
+	u, err := url.Parse(rawuri)
+	if err != nil {
+		return nil, err
+	}
+	uri := &URI{Scheme: u.Scheme}
+
+	// check the scheme is valid
+	switch uri.Scheme {
+	case "bzz", "bzzi", "bzzr":
+	default:
+		return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
+	}
+
+	// handle URIs like bzz://<addr>/<path> where the addr and path
+	// have already been split by url.Parse
+	if u.Host != "" {
+		uri.Addr = u.Host
+		uri.Path = strings.TrimLeft(u.Path, "/")
+		return uri, nil
+	}
+
+	// URI is like bzz:/<addr>/<path> so split the addr and path from
+	// the raw path (which will be /<addr>/<path>)
+	parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2)
+	uri.Addr = parts[0]
+	if len(parts) == 2 {
+		uri.Path = parts[1]
+	}
+	return uri, nil
+}
+
+func (u *URI) Raw() bool {
+	return u.Scheme == "bzzr"
+}
+
+func (u *URI) Immutable() bool {
+	return u.Scheme == "bzzi"
+}
+
+func (u *URI) String() string {
+	return u.Scheme + ":/" + u.Addr + "/" + u.Path
+}

swarm/api/uri_test.go

@@ -0,0 +1,120 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseURI(t *testing.T) {
+	type test struct {
+		uri             string
+		expectURI       *URI
+		expectErr       bool
+		expectRaw       bool
+		expectImmutable bool
+	}
+	tests := []test{
+		{
+			uri:       "",
+			expectErr: true,
+		},
+		{
+			uri:       "foo",
+			expectErr: true,
+		},
+		{
+			uri:       "bzz",
+			expectErr: true,
+		},
+		{
+			uri:       "bzz:",
+			expectURI: &URI{Scheme: "bzz"},
+		},
+		{
+			uri:             "bzzi:",
+			expectURI:       &URI{Scheme: "bzzi"},
+			expectImmutable: true,
+		},
+		{
+			uri:       "bzzr:",
+			expectURI: &URI{Scheme: "bzzr"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzz:/",
+			expectURI: &URI{Scheme: "bzz"},
+		},
+		{
+			uri:       "bzz:/abc123",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
+		},
+		{
+			uri:       "bzz:/abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
+		},
+		{
+			uri:       "bzzr:/",
+			expectURI: &URI{Scheme: "bzzr"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzzr:/abc123",
+			expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzzr:/abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzz://",
+			expectURI: &URI{Scheme: "bzz"},
+		},
+		{
+			uri:       "bzz://abc123",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
+		},
+		{
+			uri:       "bzz://abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
+		},
+	}
+	for _, x := range tests {
+		actual, err := Parse(x.uri)
+		if x.expectErr {
+			if err == nil {
+				t.Fatalf("expected %s to error", x.uri)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("error parsing %s: %s", x.uri, err)
+		}
+		if !reflect.DeepEqual(actual, x.expectURI) {
+			t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual)
+		}
+		if actual.Raw() != x.expectRaw {
+			t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw())
+		}
+		if actual.Immutable() != x.expectImmutable {
+			t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
+		}
+	}
+}

swarm/swarm.go

@@ -53,8 +53,8 @@ type Swarm struct {
 	privateKey  *ecdsa.PrivateKey
 	corsString  string
 	swapEnabled bool
-	lstore      *storage.LocalStore  // local store, needs to store for releasing resources after node stopped
-	sfs          *api.SwarmFS         // need this to cleanup all the active mounts on node exit
+	lstore      *storage.LocalStore // local store, needs to store for releasing resources after node stopped
+	sfs         *api.SwarmFS        // need this to cleanup all the active mounts on node exit
 }
 
 type SwarmAPI struct {
@@ -241,13 +241,6 @@ func (self *Swarm) Protocols() []p2p.Protocol {
 func (self *Swarm) APIs() []rpc.API {
 	return []rpc.API{
 		// public APIs
-		{
-			Namespace: "bzz",
-			Version:   "0.1",
-			Service:   api.NewStorage(self.api),
-			Public:    true,
-		},
-
 		{
 			Namespace: "bzz",
 			Version:   "0.1",
@@ -255,11 +248,6 @@ func (self *Swarm) APIs() []rpc.API {
 			Public:    true,
 		},
 		// admin APIs
-		{
-			Namespace: "bzz",
-			Version:   "0.1",
-			Service:   api.NewFileSystem(self.api),
-			Public:    false},
 		{
 			Namespace: "bzz",
 			Version:   "0.1",
@@ -278,6 +266,20 @@ func (self *Swarm) APIs() []rpc.API {
 			Service:   self.sfs,
 			Public:    false,
 		},
+		// storage APIs
+		// DEPRECATED: Use the HTTP API instead
+		{
+			Namespace: "bzz",
+			Version:   "0.1",
+			Service:   api.NewStorage(self.api),
+			Public:    true,
+		},
+		{
+			Namespace: "bzz",
+			Version:   "0.1",
+			Service:   api.NewFileSystem(self.api),
+			Public:    false,
+		},
 		// {Namespace, Version, api.NewAdmin(self), false},
 	}
 }