Metadata compression (#218)

* add metadata serialize/deserialize helpers
* support uncompressed archives in edit
* better minimal example

Author: bdon

examples/minimal.go

@@ -6,46 +6,68 @@ import (
 	"os"
 )
 
+// A program that creates the smallest meaningful PMTiles archive,
+// consisting of a purple square at tile 0,0,0 (the entire earth).
+// Uses only two library functions, SerializeHeader and SerializeEntries.
 func main() {
-	PINK := "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/D/PwAHAwL/qGeMxAAAAABJRU5ErkJggg=="
-	CYAN := "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P//PwAGBAL/VJiKjgAAAABJRU5ErkJggg=="
+	outfile, _ := os.Create("minimal.pmtiles")
+	defer outfile.Close()
 
-	pink, _ := base64.StdEncoding.DecodeString(PINK)
-	cyan, _ := base64.StdEncoding.DecodeString(CYAN)
+	// A solid purple PNG with 50% opacity.
+	png, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mM0NLxTDwADmAG/Djok1gAAAABJRU5ErkJggg==")
 
-	outfile, _ := os.Create("output.pmtiles")
+	// Create an entry with TileID=0 (z=0, x=0, y=0), Offset=0, Length=len(png), and RunLength=1.
+	entries := []pmtiles.EntryV3{{0, 0, uint32(len(png)), 1}}
 
-	var h pmtiles.HeaderV3
-	var entries []pmtiles.EntryV3
-	entries = append(entries, pmtiles.EntryV3{1, uint64(len(cyan)), uint32(len(pink)), 1})
-	entries = append(entries, pmtiles.EntryV3{2, 0, uint32(len(cyan)), 1})
+	// Create the bytes of the root directory.
 	dir := pmtiles.SerializeEntries(entries, pmtiles.NoCompression)
+
+	// the JSON metadata is the empty object.
+	metadata := "{}"
+
+	// here we set the data of the header (the first 127 bytes)
+	var h pmtiles.HeaderV3
 	h.SpecVersion = 3
-	h.RootOffset = 127
+
+	// the root directory follows the header.
+	h.RootOffset = pmtiles.HeaderV3LenBytes
 	h.RootLength = uint64(len(dir))
-	h.MetadataOffset = uint64(127 + len(dir))
-	h.MetadataLength = 2
+
+	// the JSON metadata follows the root directory.
+	h.MetadataOffset = h.RootOffset + uint64(len(dir))
+	h.MetadataLength = uint64(len(metadata))
+
+	// there are no leaves, but set the offset to the right place and length=0.
 	h.LeafDirectoryOffset = h.MetadataOffset + h.MetadataLength
 	h.LeafDirectoryLength = 0
+
+	// the tile data follows the JSON metadata.
 	h.TileDataOffset = h.LeafDirectoryOffset
-	h.TileDataLength = uint64(len(pink) + len(cyan))
-	h.AddressedTilesCount = 2
-	h.TileEntriesCount = 2
-	h.TileContentsCount = 2
-	h.InternalCompression = 1
-	h.TileCompression = 1
-	h.TileType = 2
-	h.MinZoom = 1
-	h.CenterZoom = 1
-	h.MaxZoom = 1
+	h.TileDataLength = uint64(len(png))
+
+	// set statistics
+	h.AddressedTilesCount = 1
+	h.TileEntriesCount = 1
+	h.TileContentsCount = 1
+
+	// since we store a PNG, the tile data should not be interpreted as compressed.
+	h.InternalCompression = pmtiles.NoCompression
+	h.TileCompression = pmtiles.NoCompression
+	h.TileType = pmtiles.Png
+
+	// set the zoom and geographic bounds.
+	h.MinZoom = 0
+	h.CenterZoom = 0
+	h.MaxZoom = 0
 	h.MinLatE7 = -85 * 10000000
 	h.MaxLatE7 = 85 * 10000000
 	h.MinLonE7 = -180 * 10000000
 	h.MaxLonE7 = 180 * 10000000
 
 	outfile.Write(pmtiles.SerializeHeader(h))
+
+	// write the directory, JSON metadata and tile data.
 	outfile.Write(dir)
-	outfile.Write([]byte("{}"))
-	outfile.Write(cyan)
-	outfile.Write(pink)
+	outfile.Write([]byte(metadata))
+	outfile.Write(png)
 }

pmtiles/cluster.go

@@ -2,8 +2,6 @@ package pmtiles
 
 import (
 	"bytes"
-	"compress/gzip"
-	"encoding/json"
 	"fmt"
 	"github.com/schollz/progressbar/v3"
 	"io"
@@ -28,15 +26,8 @@ func Cluster(logger *log.Logger, InputPMTiles string, deduplicate bool) error {
 	fmt.Println("total directory size", header.RootLength+header.LeafDirectoryLength)
 
 	metadataReader := io.NewSectionReader(file, int64(header.MetadataOffset), int64(header.MetadataLength))
-	var metadataBytes []byte
-	if header.InternalCompression == Gzip {
-		r, _ := gzip.NewReader(metadataReader)
-		metadataBytes, _ = io.ReadAll(r)
-	} else {
-		metadataBytes, _ = io.ReadAll(metadataReader)
-	}
-	var parsedMetadata map[string]interface{}
-	_ = json.Unmarshal(metadataBytes, &parsedMetadata)
+
+	var metadata, err = DeserializeMetadata(metadataReader, header.InternalCompression)
 
 	var CollectEntries func(uint64, uint64, func(EntryV3))
 
@@ -67,7 +58,7 @@ func Cluster(logger *log.Logger, InputPMTiles string, deduplicate bool) error {
 	})
 
 	header.Clustered = true
-	newHeader, err := finalize(logger, resolver, header, tmpfile, "output.pmtiles", parsedMetadata)
+	newHeader, err := finalize(logger, resolver, header, tmpfile, "output.pmtiles", metadata)
 	if err != nil {
 		return err
 	}

pmtiles/convert.go

@@ -375,17 +375,10 @@ func finalize(logger *log.Logger, resolve *resolver, header HeaderV3, tmpfile *o
 		logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes))/float64(resolve.AddressedTiles))
 	}
 
-	var metadataBytes []byte
-	{
-		metadataBytesUncompressed, err := json.Marshal(jsonMetadata)
-		if err != nil {
-			return header, fmt.Errorf("Failed to marshal metadata, %w", err)
-		}
-		var b bytes.Buffer
-		w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression)
-		w.Write(metadataBytesUncompressed)
-		w.Close()
-		metadataBytes = b.Bytes()
+	metadataBytes, err := SerializeMetadata(jsonMetadata, Gzip)
+
+	if err != nil {
+		return header, fmt.Errorf("Failed to marshal metadata, %w", err)
 	}
 
 	setZoomCenterDefaults(&header, resolve.Entries)

pmtiles/directory.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 )
@@ -198,6 +199,66 @@ type nopWriteCloser struct {
 
 func (w *nopWriteCloser) Close() error { return nil }
 
+func SerializeMetadata(metadata map[string]interface{}, compression Compression) ([]byte, error) {
+	jsonBytes, err := json.Marshal(metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	if compression == NoCompression {
+		return jsonBytes, nil
+	} else if compression == Gzip {
+		var b bytes.Buffer
+		w, err := gzip.NewWriterLevel(&b, gzip.BestCompression)
+		if err != nil {
+			return nil, err
+		}
+		w.Write(jsonBytes)
+		w.Close()
+		return b.Bytes(), nil
+	} else {
+		return nil, errors.New("compression not supported")
+	}
+}
+
+func DeserializeMetadataBytes(reader io.Reader, compression Compression) ([]byte, error) {
+	var jsonBytes []byte
+	var err error
+
+	if compression == NoCompression {
+		jsonBytes, err = io.ReadAll(reader)
+		if err != nil {
+			return nil, err
+		}
+	} else if compression == Gzip {
+		gzipReader, err := gzip.NewReader(reader)
+		if err != nil {
+			return nil, err
+		}
+		jsonBytes, err = io.ReadAll(gzipReader)
+		if err != nil {
+			return nil, err
+		}
+		gzipReader.Close()
+	} else {
+		return nil, errors.New("compression not supported")
+	}
+
+	return jsonBytes, nil
+}
+
+func DeserializeMetadata(reader io.Reader, compression Compression) (map[string]interface{}, error) {
+	jsonBytes, err := DeserializeMetadataBytes(reader, compression)
+	var metadata map[string]interface{}
+	err = json.Unmarshal(jsonBytes, &metadata)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return metadata, nil
+}
+
 func SerializeEntries(entries []EntryV3, compression Compression) []byte {
 	var b bytes.Buffer
 	var w io.WriteCloser

pmtiles/directory_test.go

@@ -252,3 +252,20 @@ func TestStringToCompression(t *testing.T) {
 	assert.False(t, has)
 	assert.Equal(t, "unknown", s)
 }
+
+func TestMetadataRoundtrip(t *testing.T) {
+	data := map[string]interface{}{
+		"foo": "bar",
+	}
+	b, err := SerializeMetadata(data, NoCompression)
+	assert.Nil(t, err)
+	newData, err := DeserializeMetadata(bytes.NewReader(b), NoCompression)
+	assert.Nil(t, err)
+	assert.Equal(t, "bar", newData["foo"])
+
+	b, err = SerializeMetadata(data, Gzip)
+	assert.Nil(t, err)
+	newData, err = DeserializeMetadata(bytes.NewReader(b), Gzip)
+	assert.Nil(t, err)
+	assert.Equal(t, "bar", newData["foo"])
+}

pmtiles/edit.go

@@ -2,7 +2,6 @@ package pmtiles
 
 import (
 	"bytes"
-	"compress/gzip"
 	"encoding/json"
 	"fmt"
 	"github.com/schollz/progressbar/v3"
@@ -78,22 +77,18 @@ func Edit(_ *log.Logger, inputArchive string, newHeaderJSONFile string, newMetad
 		return nil
 	}
 
-	newMetadataUncompressed, err := ioutil.ReadFile(newMetadataFile)
-
-	var parsedMetadata map[string]interface{}
-	if err := json.Unmarshal(newMetadataUncompressed, &parsedMetadata); err != nil {
+	metadataReader, err := os.Open(newMetadataFile)
+	if err != nil {
 		return err
 	}
+	defer metadataReader.Close()
 
-	var metadataBytes bytes.Buffer
-	if oldHeader.InternalCompression != Gzip {
-		return fmt.Errorf("only gzip internal compression is currently supported")
+	parsedMetadata, err := DeserializeMetadata(metadataReader, NoCompression)
+	if err != nil {
+		return err
 	}
 
-	w, _ := gzip.NewWriterLevel(&metadataBytes, gzip.BestCompression)
-	w.Write(newMetadataUncompressed)
-	w.Close()
-
+	metadataBytes, err := SerializeMetadata(parsedMetadata, oldHeader.InternalCompression)
 	if err != nil {
 		return err
 	}
@@ -111,12 +106,12 @@ func Edit(_ *log.Logger, inputArchive string, newHeaderJSONFile string, newMetad
 	defer outfile.Close()
 
 	newHeader.MetadataOffset = newHeader.RootOffset + newHeader.RootLength
-	newHeader.MetadataLength = uint64(len(metadataBytes.Bytes()))
+	newHeader.MetadataLength = uint64(len(metadataBytes))
 	newHeader.LeafDirectoryOffset = newHeader.MetadataOffset + newHeader.MetadataLength
 	newHeader.TileDataOffset = newHeader.LeafDirectoryOffset + newHeader.LeafDirectoryLength
 
 	bar := progressbar.DefaultBytes(
-		int64(HeaderV3LenBytes+newHeader.RootLength+uint64(len(metadataBytes.Bytes()))+newHeader.LeafDirectoryLength+newHeader.TileDataLength),
+		int64(HeaderV3LenBytes+newHeader.RootLength+uint64(len(metadataBytes))+newHeader.LeafDirectoryLength+newHeader.TileDataLength),
 		"writing file",
 	)
 
@@ -128,7 +123,7 @@ func Edit(_ *log.Logger, inputArchive string, newHeaderJSONFile string, newMetad
 		return err
 	}
 
-	if _, err := io.Copy(io.MultiWriter(outfile, bar), bytes.NewReader(metadataBytes.Bytes())); err != nil {
+	if _, err := io.Copy(io.MultiWriter(outfile, bar), bytes.NewReader(metadataBytes)); err != nil {
 		return err
 	}
 

pmtiles/server.go

@@ -2,7 +2,6 @@ package pmtiles
 
 import (
 	"bytes"
-	"compress/gzip"
 	"container/list"
 	"context"
 	"encoding/json"
@@ -265,14 +264,9 @@ func (server *Server) getHeaderMetadataAttempt(ctx context.Context, name, purgeE
 	}
 	defer r.Close()
 
-	var metadataBytes []byte
-	if header.InternalCompression == Gzip {
-		metadataReader, _ := gzip.NewReader(r)
-		defer metadataReader.Close()
-		metadataBytes, err = io.ReadAll(metadataReader)
-	} else if header.InternalCompression == NoCompression {
-		metadataBytes, err = io.ReadAll(r)
-	} else {
+	metadataBytes, err := DeserializeMetadataBytes(r, header.InternalCompression)
+
+	if err != nil {
 		status = "error"
 		return true, HeaderV3{}, nil, "", errors.New("unknown compression")
 	}

pmtiles/server_test.go

@@ -1,10 +1,7 @@
 package pmtiles
 
 import (
-	"bytes"
-	"compress/gzip"
 	"context"
-	"encoding/json"
 	"log"
 	"net/http"
 	"net/http/httptest"
@@ -71,20 +68,7 @@ func fakeArchive(t *testing.T, header HeaderV3, metadata map[string]interface{},
 		tileDataBytes = append(tileDataBytes, tileBytes...)
 	}
 
-	var metadataBytes []byte
-	{
-		metadataBytesUncompressed, err := json.Marshal(metadata)
-		assert.Nil(t, err)
-		var b bytes.Buffer
-		if internalCompression == Gzip {
-			w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression)
-			w.Write(metadataBytesUncompressed)
-			w.Close()
-		} else {
-			b.Write(metadataBytesUncompressed)
-		}
-		metadataBytes = b.Bytes()
-	}
+	metadataBytes, _ := SerializeMetadata(metadata, internalCompression)
 	var rootBytes []byte
 	var leavesBytes []byte
 	if leaves {