Merge pull request #362 from readium/pdf-no-meta

Add a flag to avoid the extraction of PDF metadata (except the number of pages) if they are considered too bad.

Author: llemeurfr

encrypt/process_encrypt.go

@@ -50,7 +50,7 @@ type Publication struct {
 
 // ProcessEncryption encrypts a publication
 // inputPath must contain a processable file extension.
-func ProcessEncryption(contentID, contentKey, inputPath, tempRepo, outputRepo, storageRepo, storageURL, storageFilename string, extractCover bool) (*Publication, error) {
+func ProcessEncryption(contentID, contentKey, inputPath, tempRepo, outputRepo, storageRepo, storageURL, storageFilename string, extractCover, pdfNoMeta bool) (*Publication, error) {
 
 	var pub Publication
 	pub.OutputRepo = outputRepo
@@ -141,7 +141,7 @@ func ProcessEncryption(contentID, contentKey, inputPath, tempRepo, outputRepo, s
 	case ".epub":
 		err = processEPUB(&pub, encrypter, contentKey)
 	case ".pdf":
-		err = processPDF(&pub, encrypter, contentKey)
+		err = processPDF(&pub, encrypter, contentKey, pdfNoMeta)
 	case ".lpf":
 		err = processLPF(&pub, encrypter, contentKey)
 	case ".audiobook", ".divina", ".webpub", ".rpf":
@@ -415,7 +415,7 @@ func processEPUB(pub *Publication, encrypter crypto.Encrypter, contentKey string
 }
 
 // processPDF wraps a PDF file inside a Readium Package and encrypts its resources
-func processPDF(pub *Publication, encrypter crypto.Encrypter, contentKey string) error {
+func processPDF(pub *Publication, encrypter crypto.Encrypter, contentKey string, pdfNoMeta bool) error {
 
 	log.Println("Process as PDF")
 
@@ -425,7 +425,7 @@ func processPDF(pub *Publication, encrypter crypto.Encrypter, contentKey string)
 
 	// generate a temp Readium Package (rwpp) which embeds the PDF file
 	// the first page of the PDF is extracted as a JPEG cover image
-	rwpInfo, err := pack.BuildRPFFromPDF(pub.InputPath, tmpPackagePath, coverPath)
+	rwpInfo, err := pack.BuildRPFFromPDF(pub.InputPath, tmpPackagePath, coverPath, pdfNoMeta)
 	// will will remove the tmp file even if an error is returned
 	defer os.Remove(tmpPackagePath)
 	// process error
@@ -521,6 +521,11 @@ func buildEncryptedRPF(pub *Publication, encrypter crypto.Encrypter, contentKey
 	}
 	defer reader.Close()
 
+	// set the title from the manifest if not already set
+	if pub.Title == "" {
+		pub.Title = reader.Title()
+	}
+
 	// set the target content type from the conformance type in the manifest
 	ext := filepath.Ext(pub.FileName)
 	switch reader.ConformsTo() {

frontend/webpublication/webpublication.go

@@ -108,7 +108,7 @@ func encryptPublication(inputPath string, pub *Publication, pubManager Publicati
 	// FIXME: work on a direct storage of the output file.
 	outputRepo := config.Config.FrontendServer.EncryptedRepository
 	empty := ""
-	notification, err := encrypt.ProcessEncryption(empty, empty, inputPath, empty, outputRepo, empty, empty, empty, false)
+	notification, err := encrypt.ProcessEncryption(empty, empty, inputPath, empty, outputRepo, empty, empty, empty, false, false)
 	if err != nil {
 		return err
 	}

lcpencrypt/lcpencrypt.go

@@ -21,12 +21,14 @@ func showHelpAndExit() {
 
 	fmt.Println("lcpencrypt encrypts a publication using the LCP DRM.")
 	fmt.Println("-input      source epub/pdf/lpf/audiobook file locator (file system or http GET)")
+	fmt.Println("-provider   publication provider (URI)")
 	fmt.Println("-contentid  optional, content identifier; if omitted a uuid is generated")
 	fmt.Println("-storage    optional, target location of the encrypted publication, without filename. File system path or s3 bucket")
 	fmt.Println("-url        optional, base url associated with the storage, without filename")
 	fmt.Println("-filename   optional, file name of the encrypted publication; if omitted, contentid is used")
 	fmt.Println("-temp       optional, working folder for temporary files. If not set, the current directory will be used.")
 	fmt.Println("-cover      optional, boolean, indicates that a cover should be generated")
+	fmt.Println("-pdfnometa  optional, boolean, indicates that PDF metadata must not be extracted")
 	fmt.Println("-contentkey optional, base64 encoded content key; if omitted a random content key is generated")
 	fmt.Println("-lcpsv      optional, URL, host name of the License Server to be notified; syntax http://username:[email protected]")
 	fmt.Println("-v2         optional, boolean, indicates communication with a License Server v2")
@@ -48,15 +50,16 @@ func exitWithError(context string, err error) {
 }
 
 func main() {
-	providerUri := flag.String("provider", "", "optional, provider URI")
 	inputPath := flag.String("input", "", "source epub/pdf/lpf file locator (file system or http GET)")
+	providerUri := flag.String("provider", "", "optional, publication provider (URI)")
 	contentid := flag.String("contentid", "", "optional, content identifier; if omitted, a uuid is generated")
 	storageRepo := flag.String("storage", "", "optional, target location of the encrypted publication, without filename. File system path or s3 bucket")
 	storageURL := flag.String("url", "", "optional, base url associated with the storage, without filename")
 	storageFilename := flag.String("filename", "", "optional, file name of the encrypted publication; if omitted, contentid is used")
 	outputRepo := flag.String("output", "", "optional, target folder of encrypted publications")
 	tempRepo := flag.String("temp", "", "optional, working folder for temporary files")
 	cover := flag.Bool("cover", false, "optional, boolean, indicates that covers must be generated when possible")
+	pdfnometa := flag.Bool("pdfnometa", false, "optional, boolean, indicates that PDF metadata must not be extracted")
 	contentkey := flag.String("contentkey", "", "optional, base64 encoded content key; if omitted a random content key is generated")
 	lcpsv := flag.String("lcpsv", "", "URL, host name of the License server which is notified; the preferred syntax is http://username:[email protected]")
 	v2 := flag.Bool("v2", false, "optional, boolean, indicates a v2 License serve")
@@ -101,7 +104,7 @@ func main() {
 	}
 
 	// encrypt the publication
-	publication, err := encrypt.ProcessEncryption(*contentid, *contentkey, *inputPath, *tempRepo, *outputRepo, *storageRepo, *storageURL, *storageFilename, *cover)
+	publication, err := encrypt.ProcessEncryption(*contentid, *contentkey, *inputPath, *tempRepo, *outputRepo, *storageRepo, *storageURL, *storageFilename, *cover, *pdfnometa)
 	if err != nil {
 		exitWithError("Error processing a publication", err)
 	}

pack/rwppackage.go

@@ -17,7 +17,6 @@ import (
 	"log"
 	"net/url"
 	"os"
-	"text/template"
 
 	"github.com/gen2brain/go-fitz"
 	"github.com/readium/readium-lcp-server/rwpm"
@@ -229,6 +228,11 @@ func (reader *RPFReader) ConformsTo() string {
 	return reader.manifest.Metadata.ConformsTo
 }
 
+// Title returns the title of the manifest
+func (reader *RPFReader) Title() string {
+	return reader.manifest.Metadata.Title["und"]
+}
+
 // Close closes a Readium Package Reader
 func (reader *RPFReader) Close() error {
 	return reader.zipArchive.Close()
@@ -397,7 +401,7 @@ func OpenRPF(name string) (*RPFReader, error) {
 
 // BuildRPFFromPDF builds a Readium Package (rwpp) which embeds a PDF file and a cover
 // the cover file extracted from the PDF is not deleted by this function
-func BuildRPFFromPDF(inputPath, packagePath, coverPath string) (RWPInfo, error) {
+func BuildRPFFromPDF(inputPath, packagePath, coverPath string, pdfNoMeta bool) (RWPInfo, error) {
 
 	var rwpInfo RWPInfo
 
@@ -456,69 +460,71 @@ func BuildRPFFromPDF(inputPath, packagePath, coverPath string) (RWPInfo, error)
 		return rwpInfo, err
 	}
 
-	// inject a Readium manifest into the zip output
-	manifest := `
-		{
-			"@context": "https://readium.org/webpub-manifest/context.jsonld"
-			,
-			"metadata": {
-				"@type": "http://schema.org/Book",
-				"conformsTo": "https://readium.org/webpub-manifest/profiles/pdf",
-				"title": "{{.Title}}",
-				"author": "{{.Author}}",
-				"subject": "{{.Subject}}",
-				"numberOfPages": {{.NumPages}}
-			},
-			"readingOrder": [
-				{
-					"href": "publication.pdf", "title": "publication", "type": "application/pdf"
-				}
-			],
-			"resources": [
-				{
-					"rel": "cover", "href": "cover.jpg", "type": "image/jpeg"
-				}
-			]
-		}
-		`
-
 	manifestWriter, err := zipWriter.Create(ManifestLocation)
 	if err != nil {
 		return rwpInfo, err
 	}
 
-	tmpl, err := template.New("manifest").Parse(manifest)
-	if err != nil {
-		return rwpInfo, err
+	// create simple manifest object
+	var manifest rwpm.Publication
+
+	manifest.Context.Add("https://readium.org/webpub-manifest/context.jsonld")
+	manifest.Metadata.Type = "http://schema.org/Book"
+	manifest.Metadata.ConformsTo = "https://readium.org/webpub-manifest/profiles/pdf"
+
+	// number of pages is needed to display progress in the reader
+	manifest.Metadata.NumberOfPages = rwpInfo.NumPages
+
+	// PDF metadata can be so bad that we may want to ignore them
+	if pdfNoMeta {
+		// we still need a title
+		filename := filepath.Base(inputPath)
+		rwpInfo.Title = strings.TrimSuffix(filename, filepath.Ext(filename)) // default title
+		// remove underscores, hyphens, dots which are frequent in PDF file names
+		rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "_", " ")
+		rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "-", " ")
+		rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, ".", " ")
+		rwpInfo.Title = strings.TrimSpace(rwpInfo.Title)
+		manifest.Metadata.Title.Set("und", rwpInfo.Title)
+		// add PDF metadata to the manifest
+	} else {
+		// remove underscores, hyphens, stars which are frequent in PDF titles
+		rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "_", " ")
+		rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "-", " ")
+		rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "*", " ")
+		rwpInfo.Title = strings.TrimSpace(rwpInfo.Title)
+		if rwpInfo.Title == "" {
+			rwpInfo.Title = "No Title Available" // default title
+		}
+		manifest.Metadata.Title.Set("und", rwpInfo.Title)
+		// there is zero or one author/subject in the PDF metadata
+		if len(rwpInfo.Author) != 0 {
+			manifest.Metadata.Author.AddName(rwpInfo.Author[0])
+		}
+		if len(rwpInfo.Subject) != 0 {
+			manifest.Metadata.Subject.Add(rwpm.Subject{Name: rwpInfo.Subject[0]})
+		}
 	}
 
-	// remove underscores, hyphens, stars which are frequent in PDF titles
-	rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "_", " ")
-	rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "-", " ")
-	rwpInfo.Title = strings.ReplaceAll(rwpInfo.Title, "*", " ")
-	rwpInfo.Title = strings.TrimSpace(rwpInfo.Title)
-	if rwpInfo.Title == "" {
-		rwpInfo.Title = "No Title Available" // default title
-	}
-	// there is zero or one author/subject in the PDF metadata
-	if len(rwpInfo.Author) == 0 {
-		rwpInfo.Author = []string{"unknown"}
-	}
-	if len(rwpInfo.Subject) == 0 {
-		rwpInfo.Subject = []string{"unknown"}
-	}
-	params := struct {
-		Title    string
-		Author   string
-		Subject  string
-		NumPages int
-	}{
-		Title:    rwpInfo.Title,
-		Author:   rwpInfo.Author[0],
-		Subject:  rwpInfo.Subject[0],
-		NumPages: rwpInfo.NumPages,
-	}
-	err = tmpl.Execute(manifestWriter, params)
+	manifest.ReadingOrder = []rwpm.Link{
+		{
+			Href:  "publication.pdf",
+			Title: "publication",
+			Type:  "application/pdf",
+		},
+	}
+	manifest.Resources = []rwpm.Link{
+		{
+			Rel:  []string{"cover"},
+			Href: "cover.jpg",
+			Type: "image/jpeg",
+		},
+	}
+
+	// marshal and write manifest as JSON
+	encoder := json.NewEncoder(manifestWriter)
+	encoder.SetIndent("", "  ")
+	err = encoder.Encode(manifest)
 	if err != nil {
 		return rwpInfo, err
 	}
@@ -550,8 +556,14 @@ func extractRWPInfo(inputPath, coverPath string) (RWPInfo, error) {
 	// extract PDF metadata and number of pages
 	metadata := doc.Metadata()
 	rwpInfo.Title = cleanNulls(metadata["title"])
-	rwpInfo.Author = []string{cleanNulls(metadata["author"])}
-	rwpInfo.Subject = []string{cleanNulls(metadata["subject"])}
+	author := cleanNulls(metadata["author"])
+	if author != "" {
+		rwpInfo.Author = []string{author}
+	}
+	subject := cleanNulls(metadata["subject"])
+	if subject != "" {
+		rwpInfo.Subject = []string{subject}
+	}
 	rwpInfo.NumPages = doc.NumPage()
 
 	if coverPath == "" {

pack/rwppackage_test.go

@@ -6,6 +6,7 @@ package pack
 
 import (
 	"bytes"
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -118,4 +119,66 @@ func TestSetMetadata(t *testing.T) {
 	manifest.Metadata.ReadingProgression = "ltr"
 	manifest.Metadata.Subject.Add(rwpm.Subject{Name: "software", Scheme: "iptc", Code: "04003000"})
 
+	manifest.ReadingOrder = []rwpm.Link{
+		{
+			Href: "chapter1.html",
+			Type: "application/xhtml+xml",
+		},
+		{
+			Href: "chapter2.html",
+			Type: "application/xhtml+xml",
+		},
+	}
+	manifest.Resources = []rwpm.Link{
+		{
+			Rel:  []string{"stylesheet"},
+			Href: "style.css",
+			Type: "text/css",
+		},
+	}
+
+	// simulate writing and reading back the manifest
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	err := encoder.Encode(&manifest)
+	if err != nil {
+		t.Fatalf("Could not encode manifest to JSON, %s", err)
+	}
+
+	// verify metadata
+
+	if manifest.Metadata.Identifier != "id1" {
+		t.Errorf("Expected identifier to be 'id1', got '%s'", manifest.Metadata.Identifier)
+	}
+	if title := manifest.Metadata.Title["fr"]; title != "title" {
+		t.Errorf("Expected title to be 'title', got '%s'", title)
+	}
+	if manifest.Metadata.Description != "description" {
+		t.Errorf("Expected description to be 'description', got '%s'", manifest.Metadata.Description)
+	}
+	if manifest.Metadata.Published.String() != "2020-03-05" {
+		t.Errorf("Expected published to be '2020-03-05', got '%s'", manifest.Metadata.Published.String())
+	}
+	if manifest.Metadata.Duration != 120 {
+		t.Errorf("Expected duration to be 120, got %f", manifest.Metadata.Duration)
+	}
+	if len(manifest.Metadata.Author) != 1 || manifest.Metadata.Author[0].Name["und"] != "Laurent" {
+		t.Errorf("Expected author to be 'Laurent', got '%v'", manifest.Metadata.Author)
+	}
+	if len(manifest.Metadata.Language) != 1 || manifest.Metadata.Language[0] != "fr" {
+		t.Errorf("Expected language to be 'fr', got '%v'", manifest.Metadata.Language)
+	}
+	if manifest.Metadata.ReadingProgression != "ltr" {
+		t.Errorf("Expected reading progression to be 'ltr', got '%s'", manifest.Metadata.ReadingProgression)
+	}
+	if len(manifest.Metadata.Subject) != 1 || manifest.Metadata.Subject[0].Name != "software" {
+		t.Errorf("Expected subject to be 'software', got '%v'", manifest.Metadata.Subject)
+	}
+	if len(manifest.ReadingOrder) != 2 || manifest.ReadingOrder[0].Href != "chapter1.html" {
+		t.Errorf("Expected first reading order item to be 'chapter1.html', got '%v'", manifest.ReadingOrder)
+	}
+	if len(manifest.ReadingOrder) != 2 || manifest.ReadingOrder[1].Href != "chapter2.html" {
+		t.Errorf("Expected second reading order item to be 'chapter2.html', got '%v'", manifest.ReadingOrder)
+	}
+
 }

rwpm/metadata.go

@@ -111,7 +111,7 @@ func (d Date) MarshalJSON() ([]byte, error) {
 func (d Date) String() string {
 
 	date := time.Time(d)
-	return date.Format("\"2006-01-02\"")
+	return date.Format("2006-01-02")
 }
 
 // Meta is a generic structure for other metadata