feat: async content download when creating via api (go-shiori#368)

* feat: async content download when creating via api

Invoking the content download code in a goroutine after saving the
bookmark, this way we can return a response to the user quickly while
the webpage is donwloaded and archived.

Cache api endpont (/api/cache) remains untouched until I understand
the logic behind it.

Also updated the API endpoint for the extension, though I'm unsure why
there's a difference between the "regular" API and the webext API,
they should be using the same APIs.

internal/core/processing.go

@@ -37,9 +37,9 @@ type ProcessRequest struct {
 }
 
 // ProcessBookmark process the bookmark and archive it if needed.
-// Return three values, the bookmark itself, is error fatal, and error value.
-func ProcessBookmark(req ProcessRequest) (model.Bookmark, bool, error) {
-	book := req.Bookmark
+// Return three values, is error fatal, and error value.
+func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, err error) {
+	book = req.Bookmark
 	contentType := req.ContentType
 
 	// Make sure bookmark ID is defined
@@ -59,7 +59,7 @@ func ProcessBookmark(req ProcessRequest) (model.Bookmark, bool, error) {
 		multiWriter = io.MultiWriter(archivalInput, readabilityInput, readabilityCheckInput)
 	}
 
-	_, err := io.Copy(multiWriter, req.Content)
+	_, err = io.Copy(multiWriter, req.Content)
 	if err != nil {
 		return book, false, fmt.Errorf("failed to process article: %v", err)
 	}

internal/webserver/handler-api-ext.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
 	fp "path/filepath"
@@ -92,6 +93,9 @@ func (h *handler) apiInsertViaExtension(w http.ResponseWriter, r *http.Request,
 			panic(fmt.Errorf("failed to process bookmark: %v", err))
 		}
 	}
+	if _, err := h.DB.SaveBookmarks(book); err != nil {
+		log.Printf("error saving bookmark after downloading content: %s", err)
+	}
 
 	// Save bookmark to database
 	results, err := h.DB.SaveBookmarks(book)

internal/webserver/handler-api.go

@@ -3,6 +3,7 @@ package webserver
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"math"
 	"net/http"
 	"os"
@@ -21,6 +22,27 @@ import (
 	"golang.org/x/crypto/bcrypt"
 )
 
+func downloadBookmarkContent(book *model.Bookmark, dataDir string, request *http.Request) {
+	content, contentType, err := core.DownloadBookmark(book.URL)
+	if err == nil && content != nil {
+		request := core.ProcessRequest{
+			DataDir:     dataDir,
+			Bookmark:    *book,
+			Content:     content,
+			ContentType: contentType,
+		}
+
+		result, isFatalErr, err := core.ProcessBookmark(request)
+		content.Close()
+
+		if err != nil && isFatalErr {
+			panic(fmt.Errorf("failed to process bookmark: %v", err))
+		}
+
+		book = &result
+	}
+}
+
 // apiLogin is handler for POST /api/login
 func (h *handler) apiLogin(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	// Decode request
@@ -226,17 +248,46 @@ func (h *handler) apiRenameTag(w http.ResponseWriter, r *http.Request, ps httpro
 	fmt.Fprint(w, 1)
 }
 
+// Bookmark is the record for an URL.
+type apiInsertBookmarkPayload struct {
+	URL           string      `json:"url"`
+	Title         string      `json:"title"`
+	Excerpt       string      `json:"excerpt"`
+	Tags          []model.Tag `json:"tags"`
+	CreateArchive bool        `json:"createArchive"`
+	MakePublic    int         `json:"public"`
+	Async         bool        `json:"async"`
+}
+
+// newApiInsertBookmarkPayload
+// Returns the payload struct with its defaults
+func newAPIInsertBookmarkPayload() *apiInsertBookmarkPayload {
+	return &apiInsertBookmarkPayload{
+		CreateArchive: false,
+		Async:         true,
+	}
+}
+
 // apiInsertBookmark is handler for POST /api/bookmark
 func (h *handler) apiInsertBookmark(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	// Make sure session still valid
 	err := h.validateSession(r)
 	checkError(err)
 
 	// Decode request
-	book := model.Bookmark{}
-	err = json.NewDecoder(r.Body).Decode(&book)
+	payload := newAPIInsertBookmarkPayload()
+	err = json.NewDecoder(r.Body).Decode(&payload)
 	checkError(err)
 
+	book := model.Bookmark{
+		URL:           payload.URL,
+		Title:         payload.Title,
+		Excerpt:       payload.Excerpt,
+		Tags:          payload.Tags,
+		Public:        payload.MakePublic,
+		CreateArchive: payload.CreateArchive,
+	}
+
 	// Create bookmark ID
 	book.ID, err = h.DB.CreateNewID("bookmark")
 	if err != nil {
@@ -249,37 +300,31 @@ func (h *handler) apiInsertBookmark(w http.ResponseWriter, r *http.Request, ps h
 		panic(fmt.Errorf("failed to clean URL: %v", err))
 	}
 
-	// Fetch data from internet
-	var isFatalErr bool
-	content, contentType, err := core.DownloadBookmark(book.URL)
-	if err == nil && content != nil {
-		request := core.ProcessRequest{
-			DataDir:     h.DataDir,
-			Bookmark:    book,
-			Content:     content,
-			ContentType: contentType,
-		}
-
-		book, isFatalErr, err = core.ProcessBookmark(request)
-		content.Close()
-
-		if err != nil && isFatalErr {
-			panic(fmt.Errorf("failed to process bookmark: %v", err))
-		}
-	}
-
 	// Make sure bookmark's title not empty
 	if book.Title == "" {
 		book.Title = book.URL
 	}
 
+	if !payload.Async {
+		downloadBookmarkContent(&book, h.DataDir, r)
+	}
+
 	// Save bookmark to database
 	results, err := h.DB.SaveBookmarks(book)
 	if err != nil || len(results) == 0 {
 		panic(fmt.Errorf("failed to save bookmark: %v", err))
 	}
 	book = results[0]
 
+	if payload.Async {
+		go func() {
+			downloadBookmarkContent(&book, h.DataDir, r)
+			if _, err := h.DB.SaveBookmarks(book); err != nil {
+				log.Printf("failed to save bookmark: %s", err)
+			}
+		}()
+	}
+
 	// Return the new bookmark
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(&book)