swarm/api/http: http package refactoring 1/5 and 2/5 (#17164)

swarm/api/api.go

@@ -17,6 +17,7 @@
 package api
 
 import (
+	"archive/tar"
 	"context"
 	"fmt"
 	"io"
@@ -41,22 +42,32 @@ import (
 )
 
 var (
-	apiResolveCount    = metrics.NewRegisteredCounter("api.resolve.count", nil)
-	apiResolveFail     = metrics.NewRegisteredCounter("api.resolve.fail", nil)
-	apiPutCount        = metrics.NewRegisteredCounter("api.put.count", nil)
-	apiPutFail         = metrics.NewRegisteredCounter("api.put.fail", nil)
-	apiGetCount        = metrics.NewRegisteredCounter("api.get.count", nil)
-	apiGetNotFound     = metrics.NewRegisteredCounter("api.get.notfound", nil)
-	apiGetHTTP300      = metrics.NewRegisteredCounter("api.get.http.300", nil)
-	apiModifyCount     = metrics.NewRegisteredCounter("api.modify.count", nil)
-	apiModifyFail      = metrics.NewRegisteredCounter("api.modify.fail", nil)
-	apiAddFileCount    = metrics.NewRegisteredCounter("api.addfile.count", nil)
-	apiAddFileFail     = metrics.NewRegisteredCounter("api.addfile.fail", nil)
-	apiRmFileCount     = metrics.NewRegisteredCounter("api.removefile.count", nil)
-	apiRmFileFail      = metrics.NewRegisteredCounter("api.removefile.fail", nil)
-	apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil)
-	apiAppendFileFail  = metrics.NewRegisteredCounter("api.appendfile.fail", nil)
-	apiGetInvalid      = metrics.NewRegisteredCounter("api.get.invalid", nil)
+	apiResolveCount        = metrics.NewRegisteredCounter("api.resolve.count", nil)
+	apiResolveFail         = metrics.NewRegisteredCounter("api.resolve.fail", nil)
+	apiPutCount            = metrics.NewRegisteredCounter("api.put.count", nil)
+	apiPutFail             = metrics.NewRegisteredCounter("api.put.fail", nil)
+	apiGetCount            = metrics.NewRegisteredCounter("api.get.count", nil)
+	apiGetNotFound         = metrics.NewRegisteredCounter("api.get.notfound", nil)
+	apiGetHTTP300          = metrics.NewRegisteredCounter("api.get.http.300", nil)
+	apiManifestUpdateCount = metrics.NewRegisteredCounter("api.manifestupdate.count", nil)
+	apiManifestUpdateFail  = metrics.NewRegisteredCounter("api.manifestupdate.fail", nil)
+	apiManifestListCount   = metrics.NewRegisteredCounter("api.manifestlist.count", nil)
+	apiManifestListFail    = metrics.NewRegisteredCounter("api.manifestlist.fail", nil)
+	apiDeleteCount         = metrics.NewRegisteredCounter("api.delete.count", nil)
+	apiDeleteFail          = metrics.NewRegisteredCounter("api.delete.fail", nil)
+	apiGetTarCount         = metrics.NewRegisteredCounter("api.gettar.count", nil)
+	apiGetTarFail          = metrics.NewRegisteredCounter("api.gettar.fail", nil)
+	apiUploadTarCount      = metrics.NewRegisteredCounter("api.uploadtar.count", nil)
+	apiUploadTarFail       = metrics.NewRegisteredCounter("api.uploadtar.fail", nil)
+	apiModifyCount         = metrics.NewRegisteredCounter("api.modify.count", nil)
+	apiModifyFail          = metrics.NewRegisteredCounter("api.modify.fail", nil)
+	apiAddFileCount        = metrics.NewRegisteredCounter("api.addfile.count", nil)
+	apiAddFileFail         = metrics.NewRegisteredCounter("api.addfile.fail", nil)
+	apiRmFileCount         = metrics.NewRegisteredCounter("api.removefile.count", nil)
+	apiRmFileFail          = metrics.NewRegisteredCounter("api.removefile.fail", nil)
+	apiAppendFileCount     = metrics.NewRegisteredCounter("api.appendfile.count", nil)
+	apiAppendFileFail      = metrics.NewRegisteredCounter("api.appendfile.fail", nil)
+	apiGetInvalid          = metrics.NewRegisteredCounter("api.get.invalid", nil)
 )
 
 // Resolver interface resolve a domain name to a hash using ENS
@@ -424,6 +435,185 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
 	return
 }
 
+func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Address, error) {
+	apiDeleteCount.Inc(1)
+	uri, err := Parse("bzz:/" + addr)
+	if err != nil {
+		apiDeleteFail.Inc(1)
+		return nil, err
+	}
+	key, err := a.Resolve(ctx, uri)
+
+	if err != nil {
+		return nil, err
+	}
+	newKey, err := a.UpdateManifest(ctx, key, func(mw *ManifestWriter) error {
+		log.Debug(fmt.Sprintf("removing %s from manifest %s", path, key.Log()))
+		return mw.RemoveEntry(path)
+	})
+	if err != nil {
+		apiDeleteFail.Inc(1)
+		return nil, err
+	}
+
+	return newKey, nil
+}
+
+// GetDirectoryTar fetches a requested directory as a tarstream
+// it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser
+func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) {
+	apiGetTarCount.Inc(1)
+	addr, err := a.Resolve(ctx, uri)
+	if err != nil {
+		return nil, err
+	}
+	walker, err := a.NewManifestWalker(ctx, addr, nil)
+	if err != nil {
+		apiGetTarFail.Inc(1)
+		return nil, err
+	}
+
+	piper, pipew := io.Pipe()
+
+	tw := tar.NewWriter(pipew)
+
+	go func() {
+		err := walker.Walk(func(entry *ManifestEntry) error {
+			// ignore manifests (walk will recurse into them)
+			if entry.ContentType == ManifestType {
+				return nil
+			}
+
+			// retrieve the entry's key and size
+			reader, _ := a.Retrieve(ctx, storage.Address(common.Hex2Bytes(entry.Hash)))
+			size, err := reader.Size(nil)
+			if err != nil {
+				return err
+			}
+
+			// write a tar header for the entry
+			hdr := &tar.Header{
+				Name:    entry.Path,
+				Mode:    entry.Mode,
+				Size:    size,
+				ModTime: entry.ModTime,
+				Xattrs: map[string]string{
+					"user.swarm.content-type": entry.ContentType,
+				},
+			}
+
+			if err := tw.WriteHeader(hdr); err != nil {
+				return err
+			}
+
+			// copy the file into the tar stream
+			n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
+			if err != nil {
+				return err
+			} else if n != size {
+				return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
+			}
+
+			return nil
+		})
+		if err != nil {
+			apiGetTarFail.Inc(1)
+			pipew.CloseWithError(err)
+		} else {
+			pipew.Close()
+		}
+	}()
+
+	return piper, nil
+}
+
+// GetManifestList lists the manifest entries for the specified address and prefix
+// and returns it as a ManifestList
+func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) {
+	apiManifestListCount.Inc(1)
+	walker, err := a.NewManifestWalker(ctx, addr, nil)
+	if err != nil {
+		apiManifestListFail.Inc(1)
+		return ManifestList{}, err
+	}
+
+	err = walker.Walk(func(entry *ManifestEntry) error {
+		// handle non-manifest files
+		if entry.ContentType != ManifestType {
+			// ignore the file if it doesn't have the specified prefix
+			if !strings.HasPrefix(entry.Path, prefix) {
+				return nil
+			}
+
+			// if the path after the prefix contains a slash, add a
+			// common prefix to the list, otherwise add the entry
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return nil
+			}
+			if entry.Path == "" {
+				entry.Path = "/"
+			}
+			list.Entries = append(list.Entries, entry)
+			return nil
+		}
+
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by returning nil and
+		// continuing the walk
+		if strings.HasPrefix(prefix, entry.Path) {
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a slash, add a common prefix
+		// to the list and skip the manifest, otherwise recurse into
+		// the manifest by returning nil and continuing the walk
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return ErrSkipManifest
+			}
+			return nil
+		}
+
+		// the manifest neither has the prefix or needs recursing in to
+		// so just skip it
+		return ErrSkipManifest
+	})
+
+	if err != nil {
+		apiManifestListFail.Inc(1)
+		return ManifestList{}, err
+	}
+
+	return list, nil
+}
+
+func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update func(mw *ManifestWriter) error) (storage.Address, error) {
+	apiManifestUpdateCount.Inc(1)
+	mw, err := a.NewManifestWriter(ctx, addr, nil)
+	if err != nil {
+		apiManifestUpdateFail.Inc(1)
+		return nil, err
+	}
+
+	if err := update(mw); err != nil {
+		apiManifestUpdateFail.Inc(1)
+		return nil, err
+	}
+
+	addr, err = mw.Store()
+	if err != nil {
+		apiManifestUpdateFail.Inc(1)
+		return nil, err
+	}
+	log.Debug(fmt.Sprintf("generated manifest %s", addr))
+	return addr, nil
+}
+
 // Modify loads manifest and checks the content hash before recalculating and storing the manifest.
 func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) {
 	apiModifyCount.Inc(1)
@@ -501,6 +691,43 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []
 	return fkey, newMkey.String(), nil
 }
 
+func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) {
+	apiUploadTarCount.Inc(1)
+	var contentKey storage.Address
+	tr := tar.NewReader(bodyReader)
+	defer bodyReader.Close()
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			apiUploadTarFail.Inc(1)
+			return nil, fmt.Errorf("error reading tar stream: %s", err)
+		}
+
+		// only store regular files
+		if !hdr.FileInfo().Mode().IsRegular() {
+			continue
+		}
+
+		// add the entry under the path from the request
+		manifestPath := path.Join(manifestPath, hdr.Name)
+		entry := &ManifestEntry{
+			Path:        manifestPath,
+			ContentType: hdr.Xattrs["user.swarm.content-type"],
+			Mode:        hdr.Mode,
+			Size:        hdr.Size,
+			ModTime:     hdr.ModTime,
+		}
+		contentKey, err = mw.AddEntry(ctx, tr, entry)
+		if err != nil {
+			apiUploadTarFail.Inc(1)
+			return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err)
+		}
+	}
+	return contentKey, nil
+}
+
 // RemoveFile removes a file entry in a manifest.
 func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname string, nameresolver bool) (string, error) {
 	apiRmFileCount.Inc(1)

swarm/api/client/client_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 func serverFunc(api *api.API) testutil.TestServer {
-	return swarmhttp.NewServer(api)
+	return swarmhttp.NewServer(api, "")
 }
 
 // TestClientUploadDownloadRaw test uploading and downloading raw data to swarm

swarm/api/http/error_test.go

@@ -29,7 +29,6 @@ import (
 )
 
 func TestError(t *testing.T) {
-
 	srv := testutil.NewTestSwarmServer(t, serverFunc)
 	defer srv.Close()
 

swarm/api/http/server.go

@@ -20,10 +20,8 @@ A simple http server interface to Swarm
 package http
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
-	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -68,46 +66,185 @@ var (
 	getFileCount    = metrics.NewRegisteredCounter("api.http.get.file.count", nil)
 	getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil)
 	getFileFail     = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
-	getFilesCount   = metrics.NewRegisteredCounter("api.http.get.files.count", nil)
-	getFilesFail    = metrics.NewRegisteredCounter("api.http.get.files.fail", nil)
 	getListCount    = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
 	getListFail     = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
 )
 
-// ServerConfig is the basic configuration needed for the HTTP server and also
-// includes CORS settings.
-type ServerConfig struct {
-	Addr       string
-	CorsString string
-}
-
-// browser API for registering bzz url scheme handlers:
-// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
-// electron (chromium) api for registering bzz url scheme handlers:
-// https://github.com/atom/electron/blob/master/docs/api/protocol.md
-
-// starts up http server
-func StartHTTPServer(api *api.API, config *ServerConfig) {
+func NewServer(api *api.API, corsString string) *Server {
 	var allowedOrigins []string
-	for _, domain := range strings.Split(config.CorsString, ",") {
+	for _, domain := range strings.Split(corsString, ",") {
 		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
 	}
 	c := cors.New(cors.Options{
 		AllowedOrigins: allowedOrigins,
-		AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"},
+		AllowedMethods: []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodPatch, http.MethodPut},
 		MaxAge:         600,
 		AllowedHeaders: []string{"*"},
 	})
-	hdlr := c.Handler(NewServer(api))
 
-	go http.ListenAndServe(config.Addr, hdlr)
+	mux := http.NewServeMux()
+	server := &Server{api: api}
+	mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz))
+	mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw))
+	mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable))
+	mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash))
+	mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList))
+	mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource))
+
+	mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths))
+	mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths))
+	mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths))
+
+	server.Handler = c.Handler(mux)
+	return server
+}
+func (s *Server) ListenAndServe(addr string) error {
+	return http.ListenAndServe(addr, s)
+}
+func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		if r.RequestURI == "/" {
+			if strings.Contains(r.Header.Get("Accept"), "text/html") {
+				err := landingPageTemplate.Execute(w, nil)
+				if err != nil {
+					log.Error(fmt.Sprintf("error rendering landing page: %s", err))
+				}
+				return
+			}
+			if strings.Contains(r.Header.Get("Accept"), "application/json") {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				json.NewEncoder(w).Encode("Welcome to Swarm!")
+				return
+			}
+		}
+
+		if r.URL.Path == "/robots.txt" {
+			w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+			fmt.Fprintf(w, "User-agent: *\nDisallow: /")
+			return
+		}
+		Respond(w, r, "Bad Request", http.StatusBadRequest)
+	default:
+		Respond(w, r, "Not Found", http.StatusNotFound)
+	}
+}
+func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetBzz")
+		if r.Header.Get("Accept") == "application/x-tar" {
+			reader, err := s.api.GetDirectoryTar(r.Context(), r.uri)
+			if err != nil {
+				Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
+			}
+			defer reader.Close()
+
+			w.Header().Set("Content-Type", "application/x-tar")
+			w.WriteHeader(http.StatusOK)
+			io.Copy(w, reader)
+			return
+		}
+		s.HandleGetFile(w, r)
+	case http.MethodPost:
+		log.Debug("handlePostFiles")
+		s.HandlePostFiles(w, r)
+	case http.MethodDelete:
+		log.Debug("handleBzzDelete")
+		s.HandleDelete(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetRaw")
+		s.HandleGet(w, r)
+	case http.MethodPost:
+		log.Debug("handlePostRaw")
+		s.HandlePostRaw(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetHash")
+		s.HandleGetList(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetHash")
+		s.HandleGet(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
 }
+func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetHash")
+		s.HandleGetList(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetResource")
+		s.HandleGetResource(w, r)
+	case http.MethodPost:
+		log.Debug("handlePostResource")
+		s.HandlePostResource(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
+		req := &Request{Request: *r, ruid: uuid.New()[:8]}
+		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
+		log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
+
+		// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
+		w := newLoggingResponseWriter(rw)
+		if parseBzzUri {
+			uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+			if err != nil {
+				Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
+				return
+			}
+			req.uri = uri
 
-func NewServer(api *api.API) *Server {
-	return &Server{api}
+			log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
+		}
+
+		h(w, req) // call original
+		log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
+	})
 }
 
+// browser API for registering bzz url scheme handlers:
+// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
+// electron (chromium) api for registering bzz url scheme handlers:
+// https://github.com/atom/electron/blob/master/docs/api/protocol.md
+
+// browser API for registering bzz url scheme handlers:
+// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
+// electron (chromium) api for registering bzz url scheme handlers:
+// https://github.com/atom/electron/blob/master/docs/api/protocol.md
+
 type Server struct {
+	http.Handler
 	api *api.API
 }
 
@@ -121,7 +258,7 @@ type Request struct {
 
 // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage address as a text/plain response
-func (s *Server) HandlePostRaw(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.post.raw", "ruid", r.ruid)
 
 	postRawCount.Inc(1)
@@ -148,7 +285,8 @@ func (s *Server) HandlePostRaw(ctx context.Context, w http.ResponseWriter, r *Re
 		Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest)
 		return
 	}
-	addr, _, err := s.api.Store(ctx, r.Body, r.ContentLength, toEncrypt)
+
+	addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
 	if err != nil {
 		postRawFail.Inc(1)
 		Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -167,7 +305,7 @@ func (s *Server) HandlePostRaw(ctx context.Context, w http.ResponseWriter, r *Re
 // (either a tar archive or multipart form), adds those files either to an
 // existing manifest or to a new manifest under <path> and returns the
 // resulting manifest hash as a text/plain response
-func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.post.files", "ruid", r.ruid)
 
 	postFilesCount.Inc(1)
@@ -185,7 +323,7 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r *
 
 	var addr storage.Address
 	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
-		addr, err = s.api.Resolve(ctx, r.uri)
+		addr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			postFilesFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
@@ -193,7 +331,7 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r *
 		}
 		log.Debug("resolved key", "ruid", r.ruid, "key", addr)
 	} else {
-		addr, err = s.api.NewManifest(ctx, toEncrypt)
+		addr, err = s.api.NewManifest(r.Context(), toEncrypt)
 		if err != nil {
 			postFilesFail.Inc(1)
 			Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -202,17 +340,21 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r *
 		log.Debug("new manifest", "ruid", r.ruid, "key", addr)
 	}
 
-	newAddr, err := s.updateManifest(ctx, addr, func(mw *api.ManifestWriter) error {
+	newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
 		switch contentType {
 
 		case "application/x-tar":
-			return s.handleTarUpload(ctx, r, mw)
-
+			_, err := s.handleTarUpload(r, mw)
+			if err != nil {
+				Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
+				return err
+			}
+			return nil
 		case "multipart/form-data":
-			return s.handleMultipartUpload(ctx, r, params["boundary"], mw)
+			return s.handleMultipartUpload(r, params["boundary"], mw)
 
 		default:
-			return s.handleDirectUpload(ctx, r, mw)
+			return s.handleDirectUpload(r, mw)
 		}
 	})
 	if err != nil {
@@ -228,41 +370,17 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r *
 	fmt.Fprint(w, newAddr)
 }
 
-func (s *Server) handleTarUpload(ctx context.Context, req *Request, mw *api.ManifestWriter) error {
-	log.Debug("handle.tar.upload", "ruid", req.ruid)
-	tr := tar.NewReader(req.Body)
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			return nil
-		} else if err != nil {
-			return fmt.Errorf("error reading tar stream: %s", err)
-		}
-
-		// only store regular files
-		if !hdr.FileInfo().Mode().IsRegular() {
-			continue
-		}
+func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) {
+	log.Debug("handle.tar.upload", "ruid", r.ruid)
 
-		// add the entry under the path from the request
-		path := path.Join(req.uri.Path, hdr.Name)
-		entry := &api.ManifestEntry{
-			Path:        path,
-			ContentType: hdr.Xattrs["user.swarm.content-type"],
-			Mode:        hdr.Mode,
-			Size:        hdr.Size,
-			ModTime:     hdr.ModTime,
-		}
-		log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
-		contentKey, err := mw.AddEntry(ctx, tr, entry)
-		if err != nil {
-			return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
-		}
-		log.Debug("stored content", "ruid", req.ruid, "key", contentKey)
+	key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw)
+	if err != nil {
+		return nil, err
 	}
+	return key, nil
 }
 
-func (s *Server) handleMultipartUpload(ctx context.Context, req *Request, boundary string, mw *api.ManifestWriter) error {
+func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
 	log.Debug("handle.multipart.upload", "ruid", req.ruid)
 	mr := multipart.NewReader(req.Body, boundary)
 	for {
@@ -312,7 +430,7 @@ func (s *Server) handleMultipartUpload(ctx context.Context, req *Request, bounda
 			ModTime:     time.Now(),
 		}
 		log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
-		contentKey, err := mw.AddEntry(ctx, reader, entry)
+		contentKey, err := mw.AddEntry(req.Context(), reader, entry)
 		if err != nil {
 			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
 		}
@@ -320,9 +438,9 @@ func (s *Server) handleMultipartUpload(ctx context.Context, req *Request, bounda
 	}
 }
 
-func (s *Server) handleDirectUpload(ctx context.Context, req *Request, mw *api.ManifestWriter) error {
+func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
 	log.Debug("handle.direct.upload", "ruid", req.ruid)
-	key, err := mw.AddEntry(ctx, req.Body, &api.ManifestEntry{
+	key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{
 		Path:        req.uri.Path,
 		ContentType: req.Header.Get("Content-Type"),
 		Mode:        0644,
@@ -339,24 +457,13 @@ func (s *Server) handleDirectUpload(ctx context.Context, req *Request, mw *api.M
 // HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
 // <path> from <manifest> and returns the resulting manifest hash as a
 // text/plain response
-func (s *Server) HandleDelete(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.delete", "ruid", r.ruid)
-
 	deleteCount.Inc(1)
-	key, err := s.api.Resolve(ctx, r.uri)
-	if err != nil {
-		deleteFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
-		return
-	}
-
-	newKey, err := s.updateManifest(ctx, key, func(mw *api.ManifestWriter) error {
-		log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log()), "ruid", r.ruid)
-		return mw.RemoveEntry(r.uri.Path)
-	})
+	newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path)
 	if err != nil {
 		deleteFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot update manifest: %s", err), http.StatusInternalServerError)
+		Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
 		return
 	}
 
@@ -400,7 +507,7 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) {
 // The resource name will be verbatim what is passed as the address part of the url.
 // For example, if a POST is made to /bzz-resource:/foo.eth/raw/13 a new resource with frequency 13
 // and name "foo.eth" will be created
-func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.post.resource", "ruid", r.ruid)
 	var err error
 	var addr storage.Address
@@ -429,7 +536,7 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter,
 		// we create a manifest so we can retrieve the resource with bzz:// later
 		// this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource
 		// root chunk
-		m, err := s.api.NewResourceManifest(ctx, addr.Hex())
+		m, err := s.api.NewResourceManifest(r.Context(), addr.Hex())
 		if err != nil {
 			Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
 			return
@@ -449,7 +556,7 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter,
 		// that means that we retrieve the manifest and inspect its Hash member.
 		manifestAddr := r.uri.Address()
 		if manifestAddr == nil {
-			manifestAddr, err = s.api.Resolve(ctx, r.uri)
+			manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
 			if err != nil {
 				getFail.Inc(1)
 				Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -460,7 +567,7 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter,
 		}
 
 		// get the root chunk key from the manifest
-		addr, err = s.api.ResolveResourceManifest(ctx, manifestAddr)
+		addr, err = s.api.ResolveResourceManifest(r.Context(), manifestAddr)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -519,19 +626,15 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter,
 // bzz-resource://<id>/<n> - get latest update on period n
 // bzz-resource://<id>/<n>/<m> - get update version m of period n
 // <id> = ens name or hash
-func (s *Server) HandleGetResource(ctx context.Context, w http.ResponseWriter, r *Request) {
-	s.handleGetResource(ctx, w, r)
-}
-
 // TODO: Enable pass maxPeriod parameter
-func (s *Server) handleGetResource(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.get.resource", "ruid", r.ruid)
 	var err error
 
 	// resolve the content key.
 	manifestAddr := r.uri.Address()
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(ctx, r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -542,7 +645,7 @@ func (s *Server) handleGetResource(ctx context.Context, w http.ResponseWriter, r
 	}
 
 	// get the root chunk key from the manifest
-	key, err := s.api.ResolveResourceManifest(ctx, manifestAddr)
+	key, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr)
 	if err != nil {
 		getFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -624,13 +727,13 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
 //   given storage key
 // - bzz-hash://<key> and responds with the hash of the content stored
 //   at the given storage key as a text/plain response
-func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri)
 	getCount.Inc(1)
 	var err error
 	addr := r.uri.Address()
 	if addr == nil {
-		addr, err = s.api.Resolve(ctx, r.uri)
+		addr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -645,7 +748,7 @@ func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Reques
 	// if path is set, interpret <key> as a manifest and return the
 	// raw entry at the given path
 	if r.uri.Path != "" {
-		walker, err := s.api.NewManifestWalker(ctx, addr, nil)
+		walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
@@ -693,7 +796,7 @@ func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Reques
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	reader, isEncrypted := s.api.Retrieve(ctx, addr)
+	reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
 	if _, err := reader.Size(nil); err != nil {
 		getFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
@@ -719,86 +822,10 @@ func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Reques
 	}
 }
 
-// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
-// header of "application/x-tar" and returns a tar stream of all files
-// contained in the manifest
-func (s *Server) HandleGetFiles(ctx context.Context, w http.ResponseWriter, r *Request) {
-	log.Debug("handle.get.files", "ruid", r.ruid, "uri", r.uri)
-	getFilesCount.Inc(1)
-	if r.uri.Path != "" {
-		getFilesFail.Inc(1)
-		Respond(w, r, "files request cannot contain a path", http.StatusBadRequest)
-		return
-	}
-
-	addr, err := s.api.Resolve(ctx, r.uri)
-	if err != nil {
-		getFilesFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
-		return
-	}
-	log.Debug("handle.get.files: resolved", "ruid", r.ruid, "key", addr)
-
-	walker, err := s.api.NewManifestWalker(ctx, addr, nil)
-	if err != nil {
-		getFilesFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	tw := tar.NewWriter(w)
-	defer tw.Close()
-	w.Header().Set("Content-Type", "application/x-tar")
-	w.WriteHeader(http.StatusOK)
-
-	err = walker.Walk(func(entry *api.ManifestEntry) error {
-		// ignore manifests (walk will recurse into them)
-		if entry.ContentType == api.ManifestType {
-			return nil
-		}
-
-		// retrieve the entry's key and size
-		reader, isEncrypted := s.api.Retrieve(ctx, storage.Address(common.Hex2Bytes(entry.Hash)))
-		size, err := reader.Size(nil)
-		if err != nil {
-			return err
-		}
-		w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
-
-		// write a tar header for the entry
-		hdr := &tar.Header{
-			Name:    entry.Path,
-			Mode:    entry.Mode,
-			Size:    size,
-			ModTime: entry.ModTime,
-			Xattrs: map[string]string{
-				"user.swarm.content-type": entry.ContentType,
-			},
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-
-		// copy the file into the tar stream
-		n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
-		if err != nil {
-			return err
-		} else if n != size {
-			return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
-		}
-
-		return nil
-	})
-	if err != nil {
-		getFilesFail.Inc(1)
-		log.Error(fmt.Sprintf("error generating tar stream: %s", err))
-	}
-}
-
 // HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
 // a list of all files contained in <manifest> under <path> grouped into
 // common prefixes using "/" as a delimiter
-func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri)
 	getListCount.Inc(1)
 	// ensure the root path has a trailing slash so that relative URLs work
@@ -807,7 +834,7 @@ func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Re
 		return
 	}
 
-	addr, err := s.api.Resolve(ctx, r.uri)
+	addr, err := s.api.Resolve(r.Context(), r.uri)
 	if err != nil {
 		getListFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -815,8 +842,7 @@ func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Re
 	}
 	log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr)
 
-	list, err := s.getManifestList(ctx, addr, r.uri.Path)
-
+	list, err := s.api.GetManifestList(r.Context(), addr, r.uri.Path)
 	if err != nil {
 		getListFail.Inc(1)
 		Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -846,65 +872,9 @@ func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Re
 	json.NewEncoder(w).Encode(&list)
 }
 
-func (s *Server) getManifestList(ctx context.Context, addr storage.Address, prefix string) (list api.ManifestList, err error) {
-	walker, err := s.api.NewManifestWalker(ctx, addr, nil)
-	if err != nil {
-		return
-	}
-
-	err = walker.Walk(func(entry *api.ManifestEntry) error {
-		// handle non-manifest files
-		if entry.ContentType != api.ManifestType {
-			// ignore the file if it doesn't have the specified prefix
-			if !strings.HasPrefix(entry.Path, prefix) {
-				return nil
-			}
-
-			// if the path after the prefix contains a slash, add a
-			// common prefix to the list, otherwise add the entry
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			if index := strings.Index(suffix, "/"); index > -1 {
-				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
-				return nil
-			}
-			if entry.Path == "" {
-				entry.Path = "/"
-			}
-			list.Entries = append(list.Entries, entry)
-			return nil
-		}
-
-		// if the manifest's path is a prefix of the specified prefix
-		// then just recurse into the manifest by returning nil and
-		// continuing the walk
-		if strings.HasPrefix(prefix, entry.Path) {
-			return nil
-		}
-
-		// if the manifest's path has the specified prefix, then if the
-		// path after the prefix contains a slash, add a common prefix
-		// to the list and skip the manifest, otherwise recurse into
-		// the manifest by returning nil and continuing the walk
-		if strings.HasPrefix(entry.Path, prefix) {
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			if index := strings.Index(suffix, "/"); index > -1 {
-				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
-				return api.ErrSkipManifest
-			}
-			return nil
-		}
-
-		// the manifest neither has the prefix or needs recursing in to
-		// so just skip it
-		return api.ErrSkipManifest
-	})
-
-	return list, nil
-}
-
 // HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
 // with the content of the file at <path> from the given <manifest>
-func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.get.file", "ruid", r.ruid)
 	getFileCount.Inc(1)
 	// ensure the root path has a trailing slash so that relative URLs work
@@ -916,7 +886,7 @@ func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Re
 	manifestAddr := r.uri.Address()
 
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(ctx, r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			getFileFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -927,8 +897,7 @@ func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Re
 	}
 
 	log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr)
-
-	reader, contentType, status, contentKey, err := s.api.Get(ctx, manifestAddr, r.uri.Path)
+	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path)
 
 	etag := common.Bytes2Hex(contentKey)
 	noneMatchEtag := r.Header.Get("If-None-Match")
@@ -955,8 +924,7 @@ func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Re
 	//the request results in ambiguous files
 	//e.g. /read with readme.md and readinglist.txt available in manifest
 	if status == http.StatusMultipleChoices {
-		list, err := s.getManifestList(ctx, manifestAddr, r.uri.Path)
-
+		list, err := s.api.GetManifestList(r.Context(), manifestAddr, r.uri.Path)
 		if err != nil {
 			getFileFail.Inc(1)
 			Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -1011,125 +979,6 @@ func (b bufferedReadSeeker) Seek(offset int64, whence int) (int64, error) {
 	return b.s.Seek(offset, whence)
 }
 
-func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	ctx := context.TODO()
-
-	defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
-	req := &Request{Request: *r, ruid: uuid.New()[:8]}
-	metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
-	log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
-
-	// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
-	w := newLoggingResponseWriter(rw)
-
-	if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") {
-
-		err := landingPageTemplate.Execute(w, nil)
-		if err != nil {
-			log.Error(fmt.Sprintf("error rendering landing page: %s", err))
-		}
-		return
-	}
-
-	if r.URL.Path == "/robots.txt" {
-		w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
-		fmt.Fprintf(w, "User-agent: *\nDisallow: /")
-		return
-	}
-
-	if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "application/json") {
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusOK)
-		json.NewEncoder(w).Encode("Welcome to Swarm!")
-		return
-	}
-
-	uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
-	if err != nil {
-		Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
-		return
-	}
-
-	req.uri = uri
-
-	log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
-
-	switch r.Method {
-	case "POST":
-		if uri.Raw() {
-			log.Debug("handlePostRaw")
-			s.HandlePostRaw(ctx, w, req)
-		} else if uri.Resource() {
-			log.Debug("handlePostResource")
-			s.HandlePostResource(ctx, w, req)
-		} else if uri.Immutable() || uri.List() || uri.Hash() {
-			log.Debug("POST not allowed on immutable, list or hash")
-			Respond(w, req, fmt.Sprintf("POST method on scheme %s not allowed", uri.Scheme), http.StatusMethodNotAllowed)
-		} else {
-			log.Debug("handlePostFiles")
-			s.HandlePostFiles(ctx, w, req)
-		}
-
-	case "PUT":
-		Respond(w, req, fmt.Sprintf("PUT method to %s not allowed", uri), http.StatusBadRequest)
-		return
-
-	case "DELETE":
-		if uri.Raw() {
-			Respond(w, req, fmt.Sprintf("DELETE method to %s not allowed", uri), http.StatusBadRequest)
-			return
-		}
-		s.HandleDelete(ctx, w, req)
-
-	case "GET":
-
-		if uri.Resource() {
-			s.HandleGetResource(ctx, w, req)
-			return
-		}
-
-		if uri.Raw() || uri.Hash() {
-			s.HandleGet(ctx, w, req)
-			return
-		}
-
-		if uri.List() {
-			s.HandleGetList(ctx, w, req)
-			return
-		}
-
-		if r.Header.Get("Accept") == "application/x-tar" {
-			s.HandleGetFiles(ctx, w, req)
-			return
-		}
-
-		s.HandleGetFile(ctx, w, req)
-
-	default:
-		Respond(w, req, fmt.Sprintf("%s method is not supported", r.Method), http.StatusMethodNotAllowed)
-	}
-
-	log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
-}
-
-func (s *Server) updateManifest(ctx context.Context, addr storage.Address, update func(mw *api.ManifestWriter) error) (storage.Address, error) {
-	mw, err := s.api.NewManifestWriter(ctx, addr, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := update(mw); err != nil {
-		return nil, err
-	}
-
-	addr, err = mw.Store()
-	if err != nil {
-		return nil, err
-	}
-	log.Debug(fmt.Sprintf("generated manifest %s", addr))
-	return addr, nil
-}
-
 type loggingResponseWriter struct {
 	http.ResponseWriter
 	statusCode int

swarm/api/http/server_test.go

@@ -17,6 +17,7 @@
 package http
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"crypto/rand"
@@ -24,11 +25,13 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -88,7 +91,7 @@ func TestResourcePostMode(t *testing.T) {
 }
 
 func serverFunc(api *api.API) testutil.TestServer {
-	return NewServer(api)
+	return NewServer(api, "")
 }
 
 // test the transparent resolving of multihash resource types with bzz:// scheme
@@ -356,6 +359,11 @@ func TestBzzGetPath(t *testing.T) {
 	testBzzGetPath(true, t)
 }
 
+func TestBzzTar(t *testing.T) {
+	testBzzTar(false, t)
+	testBzzTar(true, t)
+}
+
 func testBzzGetPath(encrypted bool, t *testing.T) {
 	var err error
 
@@ -592,6 +600,122 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
 	}
 }
 
+func testBzzTar(encrypted bool, t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t, serverFunc)
+	defer srv.Close()
+	fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"}
+	fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
+
+	buf := &bytes.Buffer{}
+	tw := tar.NewWriter(buf)
+	defer tw.Close()
+
+	for i, v := range fileNames {
+		size := int64(len(fileContents[i]))
+		hdr := &tar.Header{
+			Name:    v,
+			Mode:    0644,
+			Size:    size,
+			ModTime: time.Now(),
+			Xattrs: map[string]string{
+				"user.swarm.content-type": "text/plain",
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatal(err)
+		}
+
+		// copy the file into the tar stream
+		n, err := io.Copy(tw, bytes.NewBufferString(fileContents[i]))
+		if err != nil {
+			t.Fatal(err)
+		} else if n != size {
+			t.Fatal("size mismatch")
+		}
+	}
+
+	//post tar stream
+	url := srv.URL + "/bzz:/"
+	if encrypted {
+		url = url + "encrypt"
+	}
+	req, err := http.NewRequest("POST", url, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("Content-Type", "application/x-tar")
+	client := &http.Client{}
+	resp2, err := client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp2.StatusCode != http.StatusOK {
+		t.Fatalf("err %s", resp2.Status)
+	}
+	swarmHash, err := ioutil.ReadAll(resp2.Body)
+	resp2.Body.Close()
+	t.Logf("uploaded tarball successfully and got manifest address at %s", string(swarmHash))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now do a GET to get a tarball back
+	req, err = http.NewRequest("GET", fmt.Sprintf(srv.URL+"/bzz:/%s", string(swarmHash)), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("Accept", "application/x-tar")
+	resp2, err = client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp2.Body.Close()
+
+	file, err := ioutil.TempFile("", "swarm-downloaded-tarball")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(file.Name())
+	_, err = io.Copy(file, resp2.Body)
+	if err != nil {
+		t.Fatalf("error getting tarball: %v", err)
+	}
+	file.Sync()
+	file.Close()
+
+	tarFileHandle, err := os.Open(file.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(tarFileHandle)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			t.Fatalf("error reading tar stream: %s", err)
+		}
+		bb := make([]byte, hdr.Size)
+		_, err = tr.Read(bb)
+		if err != nil && err != io.EOF {
+			t.Fatal(err)
+		}
+		passed := false
+		for i, v := range fileNames {
+			if v == hdr.Name {
+				if string(bb) == fileContents[i] {
+					passed = true
+					break
+				}
+			}
+		}
+		if !passed {
+			t.Fatalf("file %s did not pass content assertion", hdr.Name)
+		}
+	}
+}
+
 // TestBzzRootRedirect tests that getting the root path of a manifest without
 // a trailing slash gets redirected to include the trailing slash so that
 // relative URLs work as expected.

swarm/swarm.go

@@ -388,10 +388,9 @@ func (self *Swarm) Start(srv *p2p.Server) error {
 	// start swarm http proxy server
 	if self.config.Port != "" {
 		addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port)
-		go httpapi.StartHTTPServer(self.api, &httpapi.ServerConfig{
-			Addr:       addr,
-			CorsString: self.config.Cors,
-		})
+		server := httpapi.NewServer(self.api, self.config.Cors)
+
+		go server.ListenAndServe(addr)
 	}
 
 	log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port))