swarm: bzz-list, bzz-raw and bzz-immutable schemes (#15667)

* swarm/api: url scheme bzz-list for getting list of files from manifest

Replace query parameter list=true for listing all files contained
in a swarm manifest with a new URL scheme bzz-list.

* swarm: replaace bzzr and bzzi schemes with bzz-raw and bzz-immutable

New URI Shemes are added and old ones are deprecated, but not removed.
Old Schemes bzzr and bzzi are functional for backward compatibility.

* swarm/api: completely remove bzzr and bzzi schemes

Remove old schemes in favour of bzz-raw and
bzz-immutable.

* swarm/api: revert "completely remove bzzr and bzzi schemes"

Keep bzzr and bzzi schemes for backward compatibility. At least
until 0.3 swarm release.

swarm/api/api.go

@@ -83,7 +83,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
 
 	// if the URI is immutable, check if the address is a hash
 	isHash := hashMatcher.MatchString(uri.Addr)
-	if uri.Immutable() {
+	if uri.Immutable() || uri.DeprecatedImmutable() {
 		if !isHash {
 			return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
 		}

swarm/api/api_test.go

@@ -216,7 +216,7 @@ func TestAPIResolve(t *testing.T) {
 			api := &Api{dns: x.dns}
 			uri := &URI{Addr: x.addr, Scheme: "bzz"}
 			if x.immutable {
-				uri.Scheme = "bzzi"
+				uri.Scheme = "bzz-immutable"
 			}
 			res, err := api.Resolve(uri)
 			if err == nil {

swarm/api/client/client.go

@@ -57,7 +57,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 	if size <= 0 {
 		return "", errors.New("data size must be greater than zero")
 	}
-	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
 	if err != nil {
 		return "", err
 	}
@@ -79,7 +79,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 
 // DownloadRaw downloads raw data from swarm
 func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
-	uri := c.Gateway + "/bzzr:/" + hash
+	uri := c.Gateway + "/bzz-raw:/" + hash
 	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
 		return nil, err
@@ -269,7 +269,7 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
 //
 // where entries ending with "/" are common prefixes.
 func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
-	res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
+	res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
 	if err != nil {
 		return nil, err
 	}

swarm/api/http/roundtripper.go

@@ -35,8 +35,8 @@ import (
 client := httpclient.New()
 // for (private) swarm proxy running locally
 client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
 
 The port you give the Roundtripper is the port the swarm proxy is listening on.
 If Host is left empty, localhost is assumed.

swarm/api/http/server.go

@@ -86,7 +86,7 @@ type Request struct {
 	uri *api.URI
 }
 
-// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage key as a text/plain response
 func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 	if r.uri.Path != "" {
@@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
 	fmt.Fprint(w, newKey)
 }
 
-// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// HandleGetRaw handles a GET request to bzz-raw://<key> and responds with
 // the raw content stored at the given storage key
 func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
@@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
 	}
 }
 
-// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
-// the "list" query parameter set to "true" and returns a list of all files
-// contained in <manifest> under <path> grouped into common prefixes using
-// "/" as a delimiter
+// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
+// a list of all files contained in <manifest> under <path> grouped into
+// common prefixes using "/" as a delimiter
 func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	// ensure the root path has a trailing slash so that relative URLs work
 	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 
@@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	if strings.Contains(r.Header.Get("Accept"), "text/html") {
 		w.Header().Set("Content-Type", "text/html")
 		err := htmlListTemplate.Execute(w, &htmlListData{
-			URI:  r.uri,
+			URI: &api.URI{
+				Scheme: "bzz",
+				Addr:   r.uri.Addr,
+				Path:   r.uri.Path,
+			},
 			List: &list,
 		})
 		if err != nil {
@@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case "POST":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandlePostRaw(w, req)
 		} else {
 			s.HandlePostFiles(w, req)
@@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		//   new manifest leaving the existing one intact, so it isn't
 		//   strictly a traditional PUT request which replaces content
 		//   at a URI, and POST is more ubiquitous)
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
 		} else {
@@ -609,25 +612,25 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 
 	case "DELETE":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
 			return
 		}
 		s.HandleDelete(w, req)
 
 	case "GET":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandleGetRaw(w, req)
 			return
 		}
 
-		if r.Header.Get("Accept") == "application/x-tar" {
-			s.HandleGetFiles(w, req)
+		if uri.List() {
+			s.HandleGetList(w, req)
 			return
 		}
 
-		if r.URL.Query().Get("list") == "true" {
-			s.HandleGetList(w, req)
+		if r.Header.Get("Accept") == "application/x-tar" {
+			s.HandleGetFiles(w, req)
 			return
 		}
 

swarm/api/http/server_test.go

@@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) {
 		wg.Wait()
 	}
 
-	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
 	if err != nil {
 		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
@@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) {
 		var resp *http.Response
 		var respbody []byte
 
-		url := srv.URL + "/bzzr:/"
+		url := srv.URL + "/bzz-raw:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}
@@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) {
 		}
 	}
 
+	for _, c := range []struct {
+		path string
+		json string
+		html string
+	}{
+		{
+			path: "/",
+			json: `{"common_prefixes":["a/"]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"a/\">a/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/",
+			json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"b/\">b/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"a\">a</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/b/",
+			json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"b\">b</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n\t<tr>\n\t  <td><a href=\"c\">c</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/x",
+		},
+		{
+			path: "",
+		},
+	} {
+		k := c.path
+		url := srv.URL + "/bzz-list:/"
+		if k[:] != "" {
+			url += common.ToHex(key[0])[2:] + "/" + k[1:]
+		}
+		t.Run("json list "+c.path, func(t *testing.T) {
+			resp, err := http.Get(url)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			body := strings.TrimSpace(string(respbody))
+			if body != c.json {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body)
+				}
+			}
+		})
+		t.Run("html list "+c.path, func(t *testing.T) {
+			req, err := http.NewRequest(http.MethodGet, url, nil)
+			if err != nil {
+				t.Fatalf("New request: %v", err)
+			}
+			req.Header.Set("Accept", "text/html")
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			if string(respbody) != c.html {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody))
+				}
+			}
+		})
+	}
+
 	nonhashtests := []string{
 		srv.URL + "/bzz:/name",
-		srv.URL + "/bzzi:/nonhash",
-		srv.URL + "/bzzr:/nonhash",
+		srv.URL + "/bzz-immutable:/nonhash",
+		srv.URL + "/bzz-raw:/nonhash",
+		srv.URL + "/bzz-list:/nonhash",
 	}
 
 	nonhashresponses := []string{
 		"error resolving name: no DNS to resolve name: &#34;name&#34;",
 		"error resolving nonhash: immutable address not a content hash: &#34;nonhash&#34;",
 		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
+		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
 	}
 
 	for i, url := range nonhashtests {

swarm/api/http/templates.go

@@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
     <tbody>
       {{ range .List.CommonPrefixes }}
 	<tr>
-	  <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+	  <td><a href="{{ basename . }}/">{{ basename . }}/</a></td>
 	  <td>DIR</td>
 	  <td>-</td>
 	</tr>

swarm/api/uri.go

@@ -26,7 +26,13 @@ import (
 type URI struct {
 	// Scheme has one of the following values:
 	//
-	// * bzz  - an entry in a swarm manifest
+	// * bzz           - an entry in a swarm manifest
+	// * bzz-raw       - raw swarm content
+	// * bzz-immutable - immutable URI of an entry in a swarm manifest
+	//                   (address is not resolved)
+	// * bzz-list      -  list of all files contained in a swarm manifest
+	//
+	// Deprecated Schemes:
 	// * bzzr - raw swarm content
 	// * bzzi - immutable URI of an entry in a swarm manifest
 	//          (address is not resolved)
@@ -50,7 +56,8 @@ type URI struct {
 // * <scheme>://<addr>
 // * <scheme>://<addr>/<path>
 //
-// with scheme one of bzz, bzzr or bzzi
+// with scheme one of bzz, bzz-raw, bzz-immutable or bzz-list
+// or deprecated ones bzzr and bzzi
 func Parse(rawuri string) (*URI, error) {
 	u, err := url.Parse(rawuri)
 	if err != nil {
@@ -60,7 +67,7 @@ func Parse(rawuri string) (*URI, error) {
 
 	// check the scheme is valid
 	switch uri.Scheme {
-	case "bzz", "bzzi", "bzzr":
+	case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzzr", "bzzi":
 	default:
 		return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
 	}
@@ -84,10 +91,22 @@ func Parse(rawuri string) (*URI, error) {
 }
 
 func (u *URI) Raw() bool {
-	return u.Scheme == "bzzr"
+	return u.Scheme == "bzz-raw"
 }
 
 func (u *URI) Immutable() bool {
+	return u.Scheme == "bzz-immutable"
+}
+
+func (u *URI) List() bool {
+	return u.Scheme == "bzz-list"
+}
+
+func (u *URI) DeprecatedRaw() bool {
+	return u.Scheme == "bzzr"
+}
+
+func (u *URI) DeprecatedImmutable() bool {
 	return u.Scheme == "bzzi"
 }
 

swarm/api/uri_test.go

@@ -23,11 +23,14 @@ import (
 
 func TestParseURI(t *testing.T) {
 	type test struct {
-		uri             string
-		expectURI       *URI
-		expectErr       bool
-		expectRaw       bool
-		expectImmutable bool
+		uri                       string
+		expectURI                 *URI
+		expectErr                 bool
+		expectRaw                 bool
+		expectImmutable           bool
+		expectList                bool
+		expectDeprecatedRaw       bool
+		expectDeprecatedImmutable bool
 	}
 	tests := []test{
 		{
@@ -47,13 +50,13 @@ func TestParseURI(t *testing.T) {
 			expectURI: &URI{Scheme: "bzz"},
 		},
 		{
-			uri:             "bzzi:",
-			expectURI:       &URI{Scheme: "bzzi"},
+			uri:             "bzz-immutable:",
+			expectURI:       &URI{Scheme: "bzz-immutable"},
 			expectImmutable: true,
 		},
 		{
-			uri:       "bzzr:",
-			expectURI: &URI{Scheme: "bzzr"},
+			uri:       "bzz-raw:",
+			expectURI: &URI{Scheme: "bzz-raw"},
 			expectRaw: true,
 		},
 		{
@@ -69,18 +72,18 @@ func TestParseURI(t *testing.T) {
 			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
 		},
 		{
-			uri:       "bzzr:/",
-			expectURI: &URI{Scheme: "bzzr"},
+			uri:       "bzz-raw:/",
+			expectURI: &URI{Scheme: "bzz-raw"},
 			expectRaw: true,
 		},
 		{
-			uri:       "bzzr:/abc123",
-			expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
+			uri:       "bzz-raw:/abc123",
+			expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
 			expectRaw: true,
 		},
 		{
-			uri:       "bzzr:/abc123/path/to/entry",
-			expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
+			uri:       "bzz-raw:/abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
 			expectRaw: true,
 		},
 		{
@@ -95,6 +98,36 @@ func TestParseURI(t *testing.T) {
 			uri:       "bzz://abc123/path/to/entry",
 			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
 		},
+		{
+			uri:        "bzz-list:",
+			expectURI:  &URI{Scheme: "bzz-list"},
+			expectList: true,
+		},
+		{
+			uri:        "bzz-list:/",
+			expectURI:  &URI{Scheme: "bzz-list"},
+			expectList: true,
+		},
+		{
+			uri:                 "bzzr:",
+			expectURI:           &URI{Scheme: "bzzr"},
+			expectDeprecatedRaw: true,
+		},
+		{
+			uri:                 "bzzr:/",
+			expectURI:           &URI{Scheme: "bzzr"},
+			expectDeprecatedRaw: true,
+		},
+		{
+			uri:                       "bzzi:",
+			expectURI:                 &URI{Scheme: "bzzi"},
+			expectDeprecatedImmutable: true,
+		},
+		{
+			uri:                       "bzzi:/",
+			expectURI:                 &URI{Scheme: "bzzi"},
+			expectDeprecatedImmutable: true,
+		},
 	}
 	for _, x := range tests {
 		actual, err := Parse(x.uri)
@@ -116,5 +149,14 @@ func TestParseURI(t *testing.T) {
 		if actual.Immutable() != x.expectImmutable {
 			t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
 		}
+		if actual.List() != x.expectList {
+			t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
+		}
+		if actual.DeprecatedRaw() != x.expectDeprecatedRaw {
+			t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw())
+		}
+		if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable {
+			t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable())
+		}
 	}
 }

swarm/dev/scripts/random-uploads.sh

@@ -46,7 +46,7 @@ main() {
 }
 
 do_random_upload() {
-  curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzzr:/"
+  curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/"
 }
 
 random_data() {