Merge branch 'master' into patch-1

.github/FUNDING.yml

@@ -0,0 +1,10 @@
+# These are supported funding model platforms
+
+open_collective: # Replace with a single Open Collective username
+ko_fi: radhifadlillah
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: ['https://www.paypal.me/RadhiFadlillah']

README.md

@@ -3,8 +3,8 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/go-shiori/shiori)](https://goreportcard.com/report/github.com/go-shiori/shiori)
 [![Docker Image](https://img.shields.io/static/v1?label=image&message=Docker&color=1488C6&logo=docker)](https://hub.docker.com/r/radhifadlillah/shiori)
 [![Deploy Heroku](https://img.shields.io/static/v1?label=deploy&message=Heroku&color=430098&logo=heroku)](https://heroku.com/deploy)
-[![Donate PayPal](https://img.shields.io/static/v1?label=donate&message=PayPal&color=00457C&logo=paypal)](https://www.paypal.me/RadhiFadlillah)
-[![Donate Ko-fi](https://img.shields.io/static/v1?label=donate&message=Ko-fi&color=F16061&logo=ko-fi)](https://ko-fi.com/radhifadlillah)
+
+**This project is now maintained by Dean Jackson (@deanishe).** The awesome original author, @RadhiFadlillah, unfortunately no longer has the time (see [#256](https://github.com/go-shiori/shiori/issues/256)), and I am honoured to take over stewardship of such a fantastic application.
 
 Shiori is a simple bookmarks manager written in the Go language. Intended as a simple clone of [Pocket](https://getpocket.com//). You can use it as a command line application or as a web application. This application is distributed as a single binary, which means it can be installed and used easily.
 
@@ -30,4 +30,4 @@ All documentation is available in [wiki](https://github.com/RadhiFadlillah/shior
 
 ## License
 
-Shiori is distributed under the terms of the [MIT license](https://choosealicense.com/licenses/mit/), which means you can use it and modify it however you want. However, if you make an enhancement for it, if possible, please send a pull request. If you like this project, please consider donating to me either via [PayPal](https://www.paypal.me/RadhiFadlillah) or [Ko-Fi](https://ko-fi.com/radhifadlillah).
+Shiori is distributed under the terms of the [MIT license](https://choosealicense.com/licenses/mit/), which means you can use it and modify it however you want. However, if you make an enhancement for it, if possible, please send a pull request.

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/disintegration/imaging v1.6.0
 	github.com/fatih/color v1.7.0
 	github.com/go-shiori/go-readability v0.0.0-20190809152430-5413e9c4ec86
+	github.com/go-shiori/warc v0.0.0-20191003110312-7b3c5582fd83
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/jmoiron/sqlx v1.2.0
@@ -21,12 +22,8 @@ require (
 	github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5
-	github.com/tdewolff/parse/v2 v2.3.7
-	go.etcd.io/bbolt v1.3.3
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	golang.org/x/image v0.0.0-20190802002840-cff245a6509b // indirect
-	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
-	golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect
 	golang.org/x/tools v0.0.0-20190809145639-6d4652c779c4 // indirect
 	google.golang.org/appengine v1.6.1 // indirect
 )

go.sum

@@ -16,8 +16,12 @@ github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-shiori/dom v0.0.0-20190930082056-9d974a4f8b25 h1:1ZfeL7TG+z4cjtC6XT+drfe23JxaVMwdqyGBh4O4foo=
+github.com/go-shiori/dom v0.0.0-20190930082056-9d974a4f8b25/go.mod h1:360KoNl36ftFYhjLHuEty78kWUGw8i1opEicvIDLfRk=
 github.com/go-shiori/go-readability v0.0.0-20190809152430-5413e9c4ec86 h1:tYq3F0DW27RniF4f5k5ACJdmOGYOmTlaXu17018YvG8=
 github.com/go-shiori/go-readability v0.0.0-20190809152430-5413e9c4ec86/go.mod h1:1tFV9uTM/xnAKQw5EgPs+ip50udKhCjaP0nYdkSDXcU=
+github.com/go-shiori/warc v0.0.0-20191003110312-7b3c5582fd83 h1:w3rI+ulrf6kMM+3313YDqxmG3r+7JctrU835wZifyL0=
+github.com/go-shiori/warc v0.0.0-20191003110312-7b3c5582fd83/go.mod h1:uaK5DAxFig7atOzy+aqLzhs6qJacMDfs8NxHV5+shzc=
 github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
@@ -85,8 +89,8 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tdewolff/parse/v2 v2.3.7 h1:DXoTUgrUE2Eap0m7zg1ljCO5C78vhEi7HTc4YnJWrRk=
-github.com/tdewolff/parse/v2 v2.3.7/go.mod h1:HansaqmN4I/U7L6/tUp0NcwT2tFO0F4EAWYGSDzkYNk=
+github.com/tdewolff/parse v2.3.4+incompatible h1:x05/cnGwIMf4ceLuDMBOdQ1qGniMoxpP46ghf0Qzh38=
+github.com/tdewolff/parse v2.3.4+incompatible/go.mod h1:8oBwCsVmUkgHO8M5iCzSIDtpzXOT0WXX9cWhz+bIzJQ=
 github.com/tdewolff/test v1.0.0 h1:jOwzqCXr5ePXEPGJaq2ivoR6HOCi+D5TPfpoyg8yvmU=
 github.com/tdewolff/test v1.0.0/go.mod h1:DiQUlutnqlEvdvhSn2LPGy4TFwRauAaYDsL+683RNX4=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
@@ -110,8 +114,8 @@ golang.org/x/net v0.0.0-20190520210107-018c4d40a106 h1:EZofHp/BzEf3j39/+7CX1JvH0
 golang.org/x/net v0.0.0-20190520210107-018c4d40a106/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 h1:qPnAdmjNA41t3QBTx2mFGf/SD1IoslhYu7AmdsVzCcs=
+golang.org/x/net v0.0.0-20190926025831-c00fd9afed17/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -122,8 +126,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190927073244-c990c680b611 h1:q9u40nxWT5zRClI/uU9dHCiYGottAg6Nzz4YUQyHxdA=
+golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

internal/cmd/open.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/go-shiori/shiori/internal/database"
-	"github.com/go-shiori/shiori/pkg/warc"
+	"github.com/go-shiori/warc"
 	"github.com/julienschmidt/httprouter"
 	"github.com/spf13/cobra"
 )

internal/cmd/root.go

@@ -122,7 +122,7 @@ func openMySQLDatabase() (database.DB, error) {
 	dbName, _ := os.LookupEnv("SHIORI_MYSQL_NAME")
 	dbAddress, _ := os.LookupEnv("SHIORI_MYSQL_ADDRESS")
 
-	connString := fmt.Sprintf("%s:%s@%s/%s", user, password, dbAddress, dbName)
+	connString := fmt.Sprintf("%s:%s@%s/%s?charset=utf8mb4", user, password, dbAddress, dbName)
 	return database.OpenMySQLDatabase(connString)
 }
 

internal/cmd/serve.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/go-shiori/shiori/internal/webserver"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -18,15 +20,40 @@ func serveCmd() *cobra.Command {
 
 	cmd.Flags().IntP("port", "p", 8080, "Port used by the server")
 	cmd.Flags().StringP("address", "a", "", "Address the server listens to")
+	cmd.Flags().StringP("webroot", "r", "/", "Root path that used by server")
 
 	return cmd
 }
 
 func serveHandler(cmd *cobra.Command, args []string) {
+	// Get flags value
 	port, _ := cmd.Flags().GetInt("port")
 	address, _ := cmd.Flags().GetString("address")
+	rootPath, _ := cmd.Flags().GetString("webroot")
+
+	// Validate root path
+	if rootPath == "" {
+		rootPath = "/"
+	}
+
+	if !strings.HasPrefix(rootPath, "/") {
+		rootPath = "/" + rootPath
+	}
+
+	if !strings.HasSuffix(rootPath, "/") {
+		rootPath += "/"
+	}
+
+	// Start server
+	serverConfig := webserver.Config{
+		DB:            db,
+		DataDir:       dataDir,
+		ServerAddress: address,
+		ServerPort:    port,
+		RootPath:      rootPath,
+	}
 
-	err := webserver.ServeApp(db, dataDir, address, port)
+	err := webserver.ServeApp(serverConfig)
 	if err != nil {
 		logrus.Fatalf("Server error: %v\n", err)
 	}

internal/core/core.go

@@ -0,0 +1,3 @@
+package core
+
+const userAgent = "Shiori/2.0.0 (+https://github.com/go-shiori/shiori)"

internal/core/download.go

@@ -18,7 +18,7 @@ func DownloadBookmark(url string) (io.ReadCloser, string, error) {
 	}
 
 	// Send download request
-	req.Header.Set("User-Agent", "Shiori/2.0.0 (+https://github.com/go-shiori/shiori)")
+	req.Header.Set("User-Agent", userAgent)
 	resp, err := httpClient.Do(req)
 	if err != nil {
 		return nil, "", err

internal/core/processing.go

@@ -18,7 +18,7 @@ import (
 	"github.com/disintegration/imaging"
 	"github.com/go-shiori/go-readability"
 	"github.com/go-shiori/shiori/internal/model"
-	"github.com/go-shiori/shiori/pkg/warc"
+	"github.com/go-shiori/warc"
 
 	// Add support for png
 	_ "image/png"
@@ -128,6 +128,7 @@ func ProcessBookmark(req ProcessRequest) (model.Bookmark, bool, error) {
 			URL:         book.URL,
 			Reader:      archivalInput,
 			ContentType: contentType,
+			UserAgent:   userAgent,
 			LogEnabled:  req.LogArchival,
 		}
 

internal/database/mysql.go

@@ -113,7 +113,7 @@ func (db *MySQLDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []mo
 	stmtInsertBook, err := tx.Preparex(`INSERT INTO bookmark
 		(id, url, title, excerpt, author, public, content, html, modified)
 		VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)
-		ON DUPLICATE KEY UPDATE 
+		ON DUPLICATE KEY UPDATE
 		url      = VALUES(url),
 		title    = VALUES(title),
 		excerpt  = VALUES(excerpt),
@@ -241,7 +241,7 @@ func (db *MySQLDatabase) GetBookmarks(opts GetBookmarksOptions) ([]model.Bookmar
 	// Add where clause for search keyword
 	if opts.Keyword != "" {
 		query += ` AND (
-			url LIKE ? OR 
+			url LIKE ? OR
 			MATCH(title, excerpt, content) AGAINST (? IN BOOLEAN MODE)
 		)`
 
@@ -329,10 +329,10 @@ func (db *MySQLDatabase) GetBookmarks(opts GetBookmarksOptions) ([]model.Bookmar
 	}
 
 	// Fetch tags for each bookmarks
-	stmtGetTags, err := db.Preparex(`SELECT t.id, t.name 
-		FROM bookmark_tag bt 
+	stmtGetTags, err := db.Preparex(`SELECT t.id, t.name
+		FROM bookmark_tag bt
 		LEFT JOIN tag t ON bt.tag_id = t.id
-		WHERE bt.bookmark_id = ? 
+		WHERE bt.bookmark_id = ?
 		ORDER BY t.name`)
 	if err != nil {
 		return nil, fmt.Errorf("failed to prepare tag query: %v", err)
@@ -369,7 +369,7 @@ func (db *MySQLDatabase) GetBookmarksCount(opts GetBookmarksOptions) (int, error
 	// Add where clause for search keyword
 	if opts.Keyword != "" {
 		query += ` AND (
-			url LIKE ? OR 
+			url LIKE ? OR
 			MATCH(title, excerpt, content) AGAINST (? IN BOOLEAN MODE)
 		)`
 
@@ -497,7 +497,7 @@ func (db *MySQLDatabase) DeleteBookmarks(ids ...int) (err error) {
 func (db *MySQLDatabase) GetBookmark(id int, url string) (model.Bookmark, bool) {
 	args := []interface{}{id}
 	query := `SELECT
-		id, url, title, excerpt, author, public, 
+		id, url, title, excerpt, author, public,
 		content, html, modified, content <> '' has_content
 		FROM bookmark WHERE id = ?`
 
@@ -562,7 +562,7 @@ func (db *MySQLDatabase) GetAccounts(opts GetAccountsOptions) ([]model.Account,
 // Returns the account and boolean whether it's exist or not.
 func (db *MySQLDatabase) GetAccount(username string) (model.Account, bool) {
 	account := model.Account{}
-	db.Get(&account, `SELECT 
+	db.Get(&account, `SELECT
 		id, username, password, owner FROM account WHERE username = ?`,
 		username)
 
@@ -603,8 +603,8 @@ func (db *MySQLDatabase) DeleteAccounts(usernames ...string) (err error) {
 // GetTags fetch list of tags and their frequency.
 func (db *MySQLDatabase) GetTags() ([]model.Tag, error) {
 	tags := []model.Tag{}
-	query := `SELECT bt.tag_id id, t.name, COUNT(bt.tag_id) n_bookmarks 
-		FROM bookmark_tag bt 
+	query := `SELECT bt.tag_id id, t.name, COUNT(bt.tag_id) n_bookmarks
+		FROM bookmark_tag bt
 		LEFT JOIN tag t ON bt.tag_id = t.id
 		GROUP BY bt.tag_id ORDER BY t.name`
 

internal/view/content.html

@@ -2,56 +2,60 @@
 <html lang="en">
 
 <head>
-	<title>$$.Title$$ - Shiori - Bookmarks Manager</title>
+	<base href="$$.RootPath$$">
+	<title>$$.Book.Title$$ - Shiori - Bookmarks Manager</title>
 
 	<meta charset="UTF-8">
 	<meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-	<link rel="apple-touch-icon-precomposed" sizes="152x152" href="/res/apple-touch-icon-152x152.png">
-	<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/res/apple-touch-icon-144x144.png">
-	<link rel="icon" type="image/png" href="/res/favicon-32x32.png" sizes="32x32">
-	<link rel="icon" type="image/png" href="/res/favicon-16x16.png" sizes="16x16">
-	<link rel="icon" type="image/x-icon" href="/res/favicon.ico">
+	<link rel="apple-touch-icon-precomposed" sizes="152x152" href="res/apple-touch-icon-152x152.png">
+	<link rel="apple-touch-icon-precomposed" sizes="144x144" href="res/apple-touch-icon-144x144.png">
+	<link rel="icon" type="image/png" href="res/favicon-32x32.png" sizes="32x32">
+	<link rel="icon" type="image/png" href="res/favicon-16x16.png" sizes="16x16">
+	<link rel="icon" type="image/x-icon" href="res/favicon.ico">
 
-	<link href="/css/source-sans-pro.min.css" rel="stylesheet">
-	<link href="/css/stylesheet.css" rel="stylesheet">
-	<link href="/css/custom-dialog.css" rel="stylesheet">
-	<link href="/css/bookmark-item.css" rel="stylesheet">
+	<link href="css/source-sans-pro.min.css" rel="stylesheet">
+	<link href="css/stylesheet.css" rel="stylesheet">
+	<link href="css/custom-dialog.css" rel="stylesheet">
+	<link href="css/bookmark-item.css" rel="stylesheet">
 
-	<script src="/js/dayjs.min.js"></script>
-	<script src="/js/vue.min.js"></script>
+	<script src="js/dayjs.min.js"></script>
+	<script src="js/vue.min.js"></script>
 </head>
 
 <body class="night">
 	<div id="content-scene" :class="{night: appOptions.nightMode}">
 		<div id="header">
 			<p id="metadata" v-cloak>Added {{localtime()}}</p>
-			<p id="title">$$.Title$$</p>
+			<p id="title">$$.Book.Title$$</p>
 			<div id="links">
-				<a href="$$.URL$$" target="_blank" rel="noopener">View Original</a>
-				$$if .HasArchive$$
-				<a href="/bookmark/$$.ID$$/archive">View Archive</a>
+				<a href="$$.Book.URL$$" target="_blank" rel="noopener">View Original</a>
+				$$if .Book.HasArchive$$
+				<a href="bookmark/$$.Book.ID$$/archive">View Archive</a>
 				$$end$$
 			</div>
 		</div>
 		<div id="content" v-pre>
-			$$html .HTML$$
+			$$html .Book.HTML$$
 		</div>
 	</div>
 
 	<script type="module">
 		// Create initial variable
-		import basePage from "/js/page/base.js";
+		import basePage from "./js/page/base.js";
 
 		new Vue({
 			el: '#content-scene',
 			mixins: [basePage],
 			data: {
-				modified: "$$.Modified$$"
+				modified: "$$.Book.Modified$$"
 			},
 			methods: {
 				localtime() {
-					var strTime = `${this.modified.replace(" ", "T")}+00:00`;
+					var strTime = this.modified.replace(" ", "T");
+					if (!strTime.endsWith("Z")) {
+						strTime += "Z";
+					}
 					return dayjs(strTime).format("D MMMM YYYY, HH:mm:ss");
 				},
 				loadSetting() {