From 6c98a50085ea07e318e3277991149c4476f0e855 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 15 Oct 2018 17:41:41 -0400 Subject: [PATCH 01/76] Add safe slug generation func in package id --- id/random.go | 12 ++++++++++++ id/random_test.go | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 id/random.go create mode 100644 id/random_test.go diff --git a/id/random.go b/id/random.go new file mode 100644 index 0000000..a34f96e --- /dev/null +++ b/id/random.go @@ -0,0 +1,12 @@ +package id + +import ( + "fmt" + "github.com/writeas/nerds/store" +) + +// GenSafeUniqueSlug generatees a reasonably unique random slug from the given +// original slug. It's "safe" because it uses 0-9 b-z excluding vowels. +func GenSafeUniqueSlug(slug string) string { + return fmt.Sprintf("%s-%s", slug, store.GenerateRandomString("0123456789bcdfghjklmnpqrstvwxyz", 4)) +} diff --git a/id/random_test.go b/id/random_test.go new file mode 100644 index 0000000..a62b0e3 --- /dev/null +++ b/id/random_test.go @@ -0,0 +1,19 @@ +package id + +import "testing" + +func TestGenSafeUniqueSlug(t *testing.T) { + slug := "slug" + r := map[string]bool{} + + for i := 0; i < 1000; i++ { + s := GenSafeUniqueSlug(slug) + if s == slug { + t.Errorf("Got same slug as inputted!") + } + if _, ok := r[s]; ok { + t.Errorf("#%d: slug %s was already generated in testing.", i, s) + } + r[s] = true + } +} From e5a5d63c6fcfbd23ede79defcc630c6e6deffbf9 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 16 Oct 2018 18:26:08 -0400 Subject: [PATCH 02/76] Add Convert(JSON|SQL) funcs --- converter/json.go | 21 +++++++++++++++++++++ converter/sql.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/converter/json.go b/converter/json.go index 6f4f682..0d97346 100644 --- a/converter/json.go +++ b/converter/json.go @@ -80,3 +80,24 @@ func (v *NullJSONString) UnmarshalJSON(data []byte) error { } return nil } + +func ConvertJSONNullString(value string) reflect.Value { + v := NullJSONString{} + if err := v.Scan(value); err != nil { + return reflect.Value{} + } + + return reflect.ValueOf(v) +} + +func ConvertJSONNullBool(value string) reflect.Value { + v := NullJSONBool{} + + if value == "on" || value == "off" { + return reflect.ValueOf(NullJSONBool{sql.NullBool{Bool: value == "on", Valid: true}}) + } + if err := v.Scan(value); err != nil { + return reflect.Value{} + } + return reflect.ValueOf(v) +} diff --git a/converter/sql.go b/converter/sql.go index 1992a81..5075106 100644 --- a/converter/sql.go +++ b/converter/sql.go @@ -40,3 +40,39 @@ func SQLNullFloat64(value string) reflect.Value { return reflect.ValueOf(v) } + +func ConvertSQLNullString(value string) reflect.Value { + v := sql.NullString{} + if err := v.Scan(value); err != nil { + return reflect.Value{} + } + + return reflect.ValueOf(v) +} + +func ConvertSQLNullBool(value string) reflect.Value { + v := sql.NullBool{} + if err := v.Scan(value); err != nil { + return reflect.Value{} + } + + return reflect.ValueOf(v) +} + +func ConvertSQLNullInt64(value string) reflect.Value { + v := sql.NullInt64{} + if err := v.Scan(value); err != nil { + return reflect.Value{} + } + + return reflect.ValueOf(v) +} + +func ConvertSQLNullFloat64(value string) reflect.Value { + v := sql.NullFloat64{} + if err := v.Scan(value); err != nil { + return reflect.Value{} + } + + return reflect.ValueOf(v) +} From 73bffef9af8c418e6fdb99fcf2f0b1ff17113780 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 16 Oct 2018 20:33:34 -0400 Subject: [PATCH 03/76] Add tags.Extract() func --- tags/tags.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tags/tags.go diff --git a/tags/tags.go b/tags/tags.go new file mode 100644 index 0000000..b980eb5 --- /dev/null +++ b/tags/tags.go @@ -0,0 +1,22 @@ +package tags + +import ( + "github.com/kylemcc/twitter-text-go/extract" +) + +func Extract(body string) []string { + matches := extract.ExtractHashtags(body) + tags := map[string]bool{} + for i := range matches { + // Second value (whether or not there's a hashtag) ignored here, since + // we're only extracting hashtags. + ht, _ := matches[i].Hashtag() + tags[ht] = true + } + + resTags := make([]string, 0) + for k := range tags { + resTags = append(resTags, k) + } + return resTags +} From 567e0c6ef0f259a6af3715a28f5c88587ecd2ea5 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 16 Oct 2018 20:38:27 -0400 Subject: [PATCH 04/76] Add i18n.LangIsRTL func --- i18n/rtl.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 i18n/rtl.go diff --git a/i18n/rtl.go b/i18n/rtl.go new file mode 100644 index 0000000..7a08e37 --- /dev/null +++ b/i18n/rtl.go @@ -0,0 +1,21 @@ +package i18n + +var rtlLangs = map[string]bool{ + "ar": true, // Arabic + "dv": true, // Divehi + "fa": true, // Persian (Farsi) + "ha": true, // Hausa + "he": true, // Hebrew + "iw": true, // Hebrew (old code) + "ji": true, // Yiddish (old code) + "ps": true, // Pashto, Pushto + "ur": true, // Urdu + "yi": true, // Yiddish +} + +func LangIsRTL(lang string) bool { + if _, ok := rtlLangs[lang]; ok { + return true + } + return false +} From 3d47537e1f2fce3182c38aacd6d477556bfc5c04 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 16 Oct 2018 22:58:06 -0400 Subject: [PATCH 05/76] Add stringmanip package --- stringmanip/runes.go | 96 ++++++++++++++++++++++++++++++++++++++++++ stringmanip/strings.go | 17 ++++++++ 2 files changed, 113 insertions(+) create mode 100644 stringmanip/runes.go create mode 100644 stringmanip/strings.go diff --git a/stringmanip/runes.go b/stringmanip/runes.go new file mode 100644 index 0000000..fd56b8e --- /dev/null +++ b/stringmanip/runes.go @@ -0,0 +1,96 @@ +package stringmanip + +/* + The MIT License (MIT) + + Copyright (c) 2016 Sergey Kamardin + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +// Source: https://github.com/gobwas/glob/blob/master/util/runes/runes.go + +func IndexRune(str string, r rune) int { + s := []rune(str) + for i, c := range s { + if c == r { + return i + } + } + return -1 +} + +func LastIndexRune(str string, needle rune) int { + s := []rune(str) + needles := []rune{needle} + ls, ln := len(s), len(needles) + + switch { + case ln == 0: + if ls == 0 { + return 0 + } + return ls + case ln == 1: + return IndexLastRune(s, needles[0]) + case ln == ls: + if EqualRunes(s, needles) { + return 0 + } + return -1 + case ln > ls: + return -1 + } + +head: + for i := ls - 1; i >= 0 && i >= ln; i-- { + for y := ln - 1; y >= 0; y-- { + if s[i-(ln-y-1)] != needles[y] { + continue head + } + } + + return i - ln + 1 + } + + return -1 +} + +func IndexLastRune(s []rune, r rune) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == r { + return i + } + } + + return -1 +} + +func EqualRunes(a, b []rune) bool { + if len(a) == len(b) { + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + + return true + } + + return false +} diff --git a/stringmanip/strings.go b/stringmanip/strings.go new file mode 100644 index 0000000..1711e4c --- /dev/null +++ b/stringmanip/strings.go @@ -0,0 +1,17 @@ +package stringmanip + +// Substring provides a safe way to extract a substring from a UTF-8 string. +// From this discussion: +// https://groups.google.com/d/msg/golang-nuts/cGq1Irv_5Vs/0SKoj49BsWQJ +func Substring(s string, p, l int) string { + if p < 0 || l <= 0 { + return "" + } + c := []rune(s) + if p > len(c) { + return "" + } else if p+l > len(c) || p+l < p { + return string(c[p:]) + } + return string(c[p : p+l]) +} From 316410bd3588524a958b5687034832f736c63a5e Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 16 Oct 2018 23:09:17 -0400 Subject: [PATCH 06/76] Don't fail test when repeat slug generated --- id/random_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/id/random_test.go b/id/random_test.go index a62b0e3..6a2b20f 100644 --- a/id/random_test.go +++ b/id/random_test.go @@ -12,7 +12,7 @@ func TestGenSafeUniqueSlug(t *testing.T) { t.Errorf("Got same slug as inputted!") } if _, ok := r[s]; ok { - t.Errorf("#%d: slug %s was already generated in testing.", i, s) + t.Logf("#%d: slug %s was already generated in testing.", i, s) } r[s] = true } From 265880c2cf09cf893ff17f7e44e3fadb254aa0d0 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sun, 28 Oct 2018 12:01:02 -0400 Subject: [PATCH 07/76] Add tags package documentation --- tags/tags.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tags/tags.go b/tags/tags.go index b980eb5..3a3c90d 100644 --- a/tags/tags.go +++ b/tags/tags.go @@ -1,9 +1,12 @@ +// Package tags supports operations around hashtags in plain text content package tags import ( "github.com/kylemcc/twitter-text-go/extract" ) +// Extract finds all hashtags in the given string and returns a de-duplicated +// list of them. func Extract(body string) []string { matches := extract.ExtractHashtags(body) tags := map[string]bool{} From 05f387ffa1b344f470983b7f326776db96f5c56d Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sun, 11 Nov 2018 11:54:21 -0500 Subject: [PATCH 08/76] Omit empty endpoints.sharedInbox This fixes writeas/writefreely#8 --- activitystreams/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitystreams/data.go b/activitystreams/data.go index 5c7cee3..956b835 100644 --- a/activitystreams/data.go +++ b/activitystreams/data.go @@ -17,7 +17,7 @@ type ( } Endpoints struct { - SharedInbox string `json:"sharedInbox"` + SharedInbox string `json:"sharedInbox,omitempty"` } Image struct { From 4cb17a6518fdbc69dc0525e3a4292ade9c8e36b5 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 24 Dec 2018 13:48:21 -0500 Subject: [PATCH 09/76] Add title dash This defaults to the em dash, but uses an en dash on German posts. Part of writeas/writefreely#1 --- l10n/phrases.go | 1 + l10n/phrases_de.go | 1 + 2 files changed, 2 insertions(+) diff --git a/l10n/phrases.go b/l10n/phrases.go index 6ff5a3d..1040635 100644 --- a/l10n/phrases.go +++ b/l10n/phrases.go @@ -32,4 +32,5 @@ var phrases = map[string]string{ "share modal title": "Share this post", "share": "share", "unpin": "unpin", + "title dash": "—", } diff --git a/l10n/phrases_de.go b/l10n/phrases_de.go index 4244737..c06e8de 100644 --- a/l10n/phrases_de.go +++ b/l10n/phrases_de.go @@ -23,4 +23,5 @@ var phrasesDE = map[string]string{ "share modal title": "Teile diesen Beitrag", "share": "Teilen", "unpin": "Lösen", + "title dash": "–", } From 5631982c2baf80f1bd00783f12bf22e26d54f0b0 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 10 Jan 2019 20:46:03 -0500 Subject: [PATCH 10/76] Support creating word-ish password This adds the NewWordish() func --- passgen/passgen.go | 14 +++++----- passgen/wordish.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 passgen/wordish.go diff --git a/passgen/passgen.go b/passgen/passgen.go index af19f55..4efe0d9 100644 --- a/passgen/passgen.go +++ b/passgen/passgen.go @@ -1,4 +1,4 @@ -// Package passgen generates random, unmemorable passwords. +// Package passgen generates random passwords. // // Example usage: // @@ -19,20 +19,20 @@ const DefLen = 20 // DefChars is the default set of characters used in the password. var DefChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()- _=+,.?/:;{}[]`~") -// New returns a random password of the default length with the default set of -// characters. +// New returns a random, unmemorable password of the default length with the +// default set of characters. func New() string { return NewLenChars(DefLen, DefChars) } -// NewLen returns a random password of the given length with the default set -// of characters. +// NewLen returns a random, unmemorable password of the given length with the +// default set of characters. func NewLen(length int) string { return NewLenChars(length, DefChars) } -// NewLenChars returns a random password of the given length with the given -// set of characters. +// NewLenChars returns a random, unmemorable password of the given length with +// the given set of characters. func NewLenChars(length int, chars []byte) string { if length == 0 { return "" diff --git a/passgen/wordish.go b/passgen/wordish.go new file mode 100644 index 0000000..4749a56 --- /dev/null +++ b/passgen/wordish.go @@ -0,0 +1,64 @@ +package passgen + +import ( + "crypto/rand" + "math/big" +) + +var ( + ar = []rune("aA4") + cr = []rune("cC") + er = []rune("eE3") + fr = []rune("fF") + gr = []rune("gG") + hr = []rune("hH") + ir = []rune("iI1") + lr = []rune("lL") + nr = []rune("nN") + or = []rune("oO0") + rr = []rune("rR") + sr = []rune("sS5") + tr = []rune("tT7") + remr = []rune("bcdfghjklmnpqrstvwxyzBCDFGHJKLMNPQRSTVWXYZ0123456789") +) + +// NewWordish generates a password made of word-like words. +func NewWordish() string { + b := []rune{} + b = append(b, randLetter(cr)) + b = append(b, randLetter(hr)) + b = append(b, randLetter(ar)) + b = append(b, randLetter(nr)) + b = append(b, randLetter(gr)) + b = append(b, randLetter(er)) + b = append(b, randLetter(tr)) + b = append(b, randLetter(hr)) + b = append(b, randLetter(ir)) + b = append(b, randLetter(sr)) + b = append(b, randLetter(ar)) + b = append(b, randLetter(fr)) + b = append(b, randLetter(tr)) + b = append(b, randLetter(er)) + b = append(b, randLetter(rr)) + b = append(b, randLetter(lr)) + b = append(b, randLetter(or)) + b = append(b, randLetter(gr)) + b = append(b, randLetter(gr)) + b = append(b, randLetter(ir)) + b = append(b, randLetter(nr)) + b = append(b, randLetter(gr)) + b = append(b, randLetter(ir)) + b = append(b, randLetter(nr)) + for i := 0; i <= 7; i++ { + b = append(b, randLetter(remr)) + } + return string(b) +} + +func randLetter(l []rune) rune { + li, err := rand.Int(rand.Reader, big.NewInt(int64(len(l)))) + if err != nil { + return rune(-1) + } + return l[li.Int64()] +} From ce8cb5ee19367662702b7057b8c3ba07b527bc48 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 10 Jan 2019 20:51:15 -0500 Subject: [PATCH 11/76] Use #writefreely badge, not Slack --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9085526..1f72735 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Write.as web core ================= [![GoDoc](https://godoc.org/github.com/writeas/web-core?status.svg)](https://godoc.org/github.com/writeas/web-core) [![Build Status](https://travis-ci.org/writeas/web-core.svg)](https://travis-ci.org/writeas/web-core) -[![Public Slack discussion](http://slack.write.as/badge.svg)](http://slack.write.as/) +[![#writefreely on freenode](https://img.shields.io/badge/freenode-%23writefreely-blue.svg)](http://webchat.freenode.net/?channels=writefreely) [![Discuss on our forum](https://img.shields.io/discourse/https/discuss.write.as/users.svg?label=forum)](https://discuss.write.as/c/development) web-core holds components of the [Write.as](https://write.as) web application. From cee61547fa671a154eba549790fe75a30f9cf4db Mon Sep 17 00:00:00 2001 From: Sam Whited Date: Fri, 1 Feb 2019 10:01:29 -0600 Subject: [PATCH 12/76] Support Go Modules Fixes #1 --- go.mod | 18 ++++++++++++++++++ go.sum | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..316a321 --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module github.com/writeas/web-core + +go 1.10 + +require ( + github.com/kr/pretty v0.1.0 // indirect + github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec + github.com/microcosm-cc/bluemonday v1.0.2 + github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/writeas/impart v1.1.0 + github.com/writeas/nerds v1.0.0 + github.com/writeas/openssl-go v1.0.0 + github.com/writeas/saturday v1.6.0 + golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c0585e9 --- /dev/null +++ b/go.sum @@ -0,0 +1,29 @@ +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec h1:ZXWuspqypleMuJy4bzYEqlMhJnGAYpLrWe5p7W3CdvI= +github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec/go.mod h1:voECJzdraJmolzPBgL9Z7ANwXf4oMXaTCsIkdiPpR/g= +github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= +github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= +github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= +github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo= +github.com/writeas/nerds v1.0.0/go.mod h1:Gn2bHy1EwRcpXeB7ZhVmuUwiweK0e+JllNf66gvNLdU= +github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjFc5o= +github.com/writeas/openssl-go v1.0.0/go.mod h1:WsKeK5jYl0B5y8ggOmtVjbmb+3rEGqSD25TppjJnETA= +github.com/writeas/saturday v1.6.0 h1:HNUtX8TVJJnSdxE8vaAgtHiAJVt+Of5AJYlm62pmVlI= +github.com/writeas/saturday v1.6.0/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= From ad1a4272b06ff4730627cdffae6cb9944ce7d002 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 4 Apr 2019 16:29:00 -0400 Subject: [PATCH 13/76] Support new commentsEnabled property This adds a CommentsEnabled field to the activitystreams.Object struct, as well as the Extensions needed in the Context field in order to describe the field. This is a field previously supported by PeerTube, and just recently added in PixelFed. --- activitystreams/activity.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index 13ffcee..e9097d0 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -11,6 +11,11 @@ const ( toPublic = "https://www.w3.org/ns/activitystreams#Public" ) +var Extensions = map[string]string{ + "sc": "http://schema.org#", + "commentsEnabled": "sc:Boolean", +} + // Activity describes actions that have either already occurred, are in the // process of occurring, or may occur in the future. type Activity struct { @@ -113,6 +118,9 @@ type Object struct { Content string `json:"content"` ContentMap map[string]string `json:"contentMap,omitempty"` Tag []Tag `json:"tag"` + + // Extensions + CommentsEnabled bool `json:"commentsEnabled"` } // NewNoteObject creates a basic Note object that includes the public From 528e59e922d60909246d0ff03245b9858b294332 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 4 Apr 2019 16:35:50 -0400 Subject: [PATCH 14/76] Update README to explain use in WriteFreely --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f72735..6a882d9 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,4 @@ Write.as web core [![#writefreely on freenode](https://img.shields.io/badge/freenode-%23writefreely-blue.svg)](http://webchat.freenode.net/?channels=writefreely) [![Discuss on our forum](https://img.shields.io/discourse/https/discuss.write.as/users.svg?label=forum)](https://discuss.write.as/c/development) -web-core holds components of the [Write.as](https://write.as) web application. +web-core holds components of the [WriteFreely](https://writefreely.org) web application, and is shared with other [Write.as](https://write.as) products, like [Snap.as](https://snap.as). From 7b4bf5c1b26a20c5dff457f10e9a7d2b40a31d21 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 4 Apr 2019 16:57:46 -0400 Subject: [PATCH 15/76] Include extensions when building Create & Update activities --- activitystreams/activity.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index e9097d0..0dec0c1 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -43,6 +43,7 @@ func NewCreateActivity(o *Object) *Activity { BaseObject: BaseObject{ Context: []interface{}{ Namespace, + Extensions, }, ID: o.ID, Type: "Create", @@ -61,6 +62,7 @@ func NewUpdateActivity(o *Object) *Activity { BaseObject: BaseObject{ Context: []interface{}{ Namespace, + Extensions, }, ID: o.ID, Type: "Update", From c5dba02ee6be9fad7399789f5211d55a435e4d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gytis=20Repe=C4=8Dka?= Date: Tue, 7 May 2019 22:17:05 +0300 Subject: [PATCH 16/76] Added Lithuanian localization. --- l10n/phrases_lt.go | 27 +++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 l10n/phrases_lt.go diff --git a/l10n/phrases_lt.go b/l10n/phrases_lt.go new file mode 100644 index 0000000..acd839f --- /dev/null +++ b/l10n/phrases_lt.go @@ -0,0 +1,27 @@ +package l10n + +var phrasesLT = map[string]string{ + "Anonymous post": "Anoniminis įrašas", + "Blogs": "Tinklaraščiai", + "Enter": "Įvesti", + "Newer": "Naujesni", + "Older": "Senesni", + "Posts": "Įrašai", + "Publish to...": "Publikuoti į...", + "Publish": "Publikuoti", + "Read more...": "Skaityti daugiau...", + "This blog requires a password.": "Šis tinklaraštis reikalauja slaptažodžio.", + "Toggle theme": "Keisti temą", + "View posts": "Peržiūrėti įrašus", + "delete": "ištrinti", + "edit": "koreguoti", + "move to...": "perkelti į...", + "pin": "prisegti", + "published with write.as": "publikuojama su write.as", + "share modal ending": "Siųskite draugui, dalinkitės internete ar per Twitter. Sužinokite daugiau.", + "share modal introduction": "Kiekvienas publikuotas įrašas turi slaptą, unikalų URL kuriuo galite pasidalinti su bet kuo. Šis URL yra:", + "share modal title": "Dalintis šiuo įrašu", + "share": "dalintis", + "unpin": "atsegti", + "title dash": "–", +} diff --git a/l10n/strings.go b/l10n/strings.go index 4c85091..edf9eb0 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -24,6 +24,8 @@ func Strings(lang string) map[string]string { return phrasesIT case "ja": return phrasesJA + case "lt": + return phrasesLT case "mk": return phrasesMK case "pl": From 1f29da565f2ee34f5937ed437951c8e4845b0b1b Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 23 May 2019 06:59:22 -0400 Subject: [PATCH 17/76] Include calling filename and line # in error logs ...instead of the filename and line number of the Printf call in log.Error() --- log/log.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/log/log.go b/log/log.go index ec37124..dac5925 100644 --- a/log/log.go +++ b/log/log.go @@ -2,8 +2,10 @@ package log import ( + "fmt" "log" "os" + "runtime" ) var ( @@ -13,7 +15,7 @@ var ( func init() { InfoLog = log.New(os.Stdout, "", log.Ldate|log.Ltime) - ErrorLog = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile) + ErrorLog = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime) } // Info logs an informational message to Stdout. @@ -23,5 +25,18 @@ func Info(s string, v ...interface{}) { // Error logs an error to Stderr. func Error(s string, v ...interface{}) { - ErrorLog.Printf(s, v...) + // Include original caller information + _, file, line, _ := runtime.Caller(1) + + // Determine short filename (from standard log package) + short := file + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + short = file[i+1:] + break + } + } + file = short + + ErrorLog.Printf(fmt.Sprintf("%s:%d: ", short, line)+s, v...) } From 46304124ec6d59daf691877182fa8040508f0cea Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 30 May 2019 10:12:34 -0400 Subject: [PATCH 18/76] Document posts.ExtractTitle func --- posts/parse.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/posts/parse.go b/posts/parse.go index 85d0c4d..b082a05 100644 --- a/posts/parse.go +++ b/posts/parse.go @@ -4,6 +4,8 @@ import ( "strings" ) +// ExtractTitle takes the given raw post text and returns a title, if explicitly +// provided, and a body. func ExtractTitle(content string) (title string, body string) { if hashIndex := strings.Index(content, "# "); hashIndex == 0 { eol := strings.IndexRune(content, '\n') From 35ecbe8f09b47c5cdbe010fa9af0cec662698bb8 Mon Sep 17 00:00:00 2001 From: Wexpo Lyu Date: Sat, 26 Oct 2019 17:42:19 +0800 Subject: [PATCH 19/76] Update Chinese translation. - Make sentences concise. - Implement spaces between Chinese and Western characters. --- l10n/phrases_zh.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/l10n/phrases_zh.go b/l10n/phrases_zh.go index a5e023c..4a4ec25 100644 --- a/l10n/phrases_zh.go +++ b/l10n/phrases_zh.go @@ -3,24 +3,24 @@ package l10n var phrasesZH = map[string]string{ "Anonymous post": "匿名文章", "Blogs": "博客", - "Enter": "按下Enter按钮", - "Newer": "最近的博客", - "Older": "之前的博客", + "Enter": "按下回车", + "Newer": "更新", + "Older": "较旧", "Posts": "文章", "Publish to...": "发布到...", "Publish": "发布", "Read more...": "阅读更多", - "This blog requires a password.": "这篇博客需要密码", + "This blog requires a password.": "这篇文章需要密码", "Toggle theme": "更换主题", "View posts": "查看文章", "delete": "删除", "edit": "编辑", "move to...": "移动到", "pin": "固定文章", - "published with write.as": "用write.as来发布", - "share modal ending": "发送给朋友，或者线上分享，如果可能的话，分享到推特上。了解更多。", - "share modal introduction": "发布的每篇文章都有一个唯一的私有URL，你可以把它分享给任何人。这是URL：", - "share modal title": "分享这篇文章", + "published with write.as": "通过 write.as 发布", + "share modal ending": "发送给朋友、与世界分享或者发条推。了解更多。", + "share modal introduction": "发布的每篇文章都有一个唯一的私有网址，你可以把它分享给任何人：", + "share modal title": "分享文章", "share": "分享", "unpin": "取消固定", } From 45b1985b977c1b957f3fd2985518a172301c9667 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 11 Nov 2019 21:48:19 +0900 Subject: [PATCH 20/76] Change Write.as -> WriteFreely in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6a882d9..152167f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -Write.as web core -================= +WriteFreely web core +==================== [![GoDoc](https://godoc.org/github.com/writeas/web-core?status.svg)](https://godoc.org/github.com/writeas/web-core) [![Build Status](https://travis-ci.org/writeas/web-core.svg)](https://travis-ci.org/writeas/web-core) [![#writefreely on freenode](https://img.shields.io/badge/freenode-%23writefreely-blue.svg)](http://webchat.freenode.net/?channels=writefreely) From f54ee176fc137e725dd90edd9563d4f11c216040 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 11 Nov 2019 21:56:08 +0900 Subject: [PATCH 21/76] Remove commentsEnabled AP extension This isn't used. --- activitystreams/activity.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index 0dec0c1..83f87b7 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -11,10 +11,7 @@ const ( toPublic = "https://www.w3.org/ns/activitystreams#Public" ) -var Extensions = map[string]string{ - "sc": "http://schema.org#", - "commentsEnabled": "sc:Boolean", -} +var Extensions = map[string]string{} // Activity describes actions that have either already occurred, are in the // process of occurring, or may occur in the future. @@ -122,7 +119,7 @@ type Object struct { Tag []Tag `json:"tag"` // Extensions - CommentsEnabled bool `json:"commentsEnabled"` + // NOTE: add extensions here } // NewNoteObject creates a basic Note object that includes the public From 68a680d1b03c895b0dfda763a8509d027ade02b8 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 13 Aug 2020 12:17:34 -0400 Subject: [PATCH 22/76] Add silobridge package --- silobridge/silobridge.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 silobridge/silobridge.go diff --git a/silobridge/silobridge.go b/silobridge/silobridge.go new file mode 100644 index 0000000..1cc0d2d --- /dev/null +++ b/silobridge/silobridge.go @@ -0,0 +1,29 @@ +package silobridge + +// fakeAPInstances contains a list of sites that we allow writers to mention +// with the @handle@instance.tld syntax, plus the corresponding prefix to +// insert between `https://instance.tld/` and `handle` (e.g. +// https://medium.com/@handle) +var fakeAPInstances = map[string]string{ + "deviantart.com": "", + "facebook.com": "", + "flickr.com": "photos/", + "github.com": "", + "instagram.com": "", + "medium.com": "@", + "reddit.com": "user/", + "twitter.com": "", + "wattpad.com": "user/", + "youtube.com": "user/", +} + +// Profile returns the full profile URL for a fake ActivityPub instance, based +// on the given handle and domain. If the domain isn't recognized, an empty +// string is returned. +func Profile(handle, domain string) string { + prefix, ok := fakeAPInstances[domain] + if !ok { + return "" + } + return "https://" + domain + "/" + prefix + handle +} From 83e2689111b4e421127050dc1b1be4677f7dbbea Mon Sep 17 00:00:00 2001 From: Edd Salkield Date: Tue, 22 Sep 2020 22:06:36 +0000 Subject: [PATCH 23/76] Migrate to new uuid library --- auth/auth.go | 6 +++--- go.mod | 2 +- go.sum | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 868897c..f64446d 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -1,7 +1,7 @@ package auth import ( - uuid "github.com/nu7hatch/gouuid" + uuid "github.com/gofrs/uuid" "github.com/writeas/web-core/log" "strings" ) @@ -16,7 +16,7 @@ func GetToken(header string) []byte { token = f[1] } } - t, err := uuid.ParseHex(token) + t, err := uuid.FromString(token) if err != nil { log.Error("Couldn't parseHex on '%s': %v", accessToken, err) } else { @@ -31,7 +31,7 @@ func GetHeaderToken(header string) []byte { if len(header) > 0 { f := strings.Fields(header) if len(f) == 2 && f[0] == "Token" { - t, err := uuid.ParseHex(f[1]) + t, err := uuid.FromString(f[1]) if err != nil { log.Error("Couldn't parseHex on '%s': %v", accessToken, err) } else { diff --git a/go.mod b/go.mod index 316a321..dde8251 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/writeas/web-core go 1.10 require ( + github.com/gofrs/uuid v3.3.0+incompatible github.com/kr/pretty v0.1.0 // indirect github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec github.com/microcosm-cc/bluemonday v1.0.2 - github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/writeas/impart v1.1.0 github.com/writeas/nerds v1.0.0 diff --git a/go.sum b/go.sum index c0585e9..4024058 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,6 @@ +github.com/gofrs/uuid v1.2.0 h1:coDhrjgyJaglxSjxuJdqQSSdUpG3w6p1OwN2od6frBU= +github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -7,8 +10,6 @@ github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec h1:ZXWuspq github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec/go.mod h1:voECJzdraJmolzPBgL9Z7ANwXf4oMXaTCsIkdiPpR/g= github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= -github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= From 339af195c2c281cff240296b6609e0b3216d5218 Mon Sep 17 00:00:00 2001 From: Edd Salkield Date: Tue, 22 Sep 2020 22:14:10 +0000 Subject: [PATCH 24/76] Remove dependency on github.com/writeas/nerds --- go.mod | 1 - go.sum | 2 -- id/random.go | 18 ++++++++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 316a321..90d1e26 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/writeas/impart v1.1.0 - github.com/writeas/nerds v1.0.0 github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.6.0 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 diff --git a/go.sum b/go.sum index c0585e9..afd6e1b 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= -github.com/writeas/nerds v1.0.0 h1:ZzRcCN+Sr3MWID7o/x1cr1ZbLvdpej9Y1/Ho+JKlqxo= -github.com/writeas/nerds v1.0.0/go.mod h1:Gn2bHy1EwRcpXeB7ZhVmuUwiweK0e+JllNf66gvNLdU= github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjFc5o= github.com/writeas/openssl-go v1.0.0/go.mod h1:WsKeK5jYl0B5y8ggOmtVjbmb+3rEGqSD25TppjJnETA= github.com/writeas/saturday v1.6.0 h1:HNUtX8TVJJnSdxE8vaAgtHiAJVt+Of5AJYlm62pmVlI= diff --git a/id/random.go b/id/random.go index a34f96e..0c804df 100644 --- a/id/random.go +++ b/id/random.go @@ -2,11 +2,25 @@ package id import ( "fmt" - "github.com/writeas/nerds/store" + "crypto/rand" ) +// GenerateRandomString creates a random string of characters of the given +// length from the given dictionary of possible characters. +// +// This example generates a hexadecimal string 6 characters long: +// GenerateRandomString("0123456789abcdef", 6) +func GenerateRandomString(dictionary string, l int) string { + var bytes = make([]byte, l) + rand.Read(bytes) + for k, v := range bytes { + bytes[k] = dictionary[v%byte(len(dictionary))] + } + return string(bytes) +} + // GenSafeUniqueSlug generatees a reasonably unique random slug from the given // original slug. It's "safe" because it uses 0-9 b-z excluding vowels. func GenSafeUniqueSlug(slug string) string { - return fmt.Sprintf("%s-%s", slug, store.GenerateRandomString("0123456789bcdfghjklmnpqrstvwxyz", 4)) + return fmt.Sprintf("%s-%s", slug, GenerateRandomString("0123456789bcdfghjklmnpqrstvwxyz", 4)) } From cee889bd5865956c738ae0b03063e7c6be426629 Mon Sep 17 00:00:00 2001 From: Colin Axner Date: Mon, 23 Nov 2020 15:36:27 +0100 Subject: [PATCH 25/76] Add Reset func to memo Add a new function Reset to memo to forcibly reset the cache --- memo/memo.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/memo/memo.go b/memo/memo.go index a31c8be..893bc79 100644 --- a/memo/memo.go +++ b/memo/memo.go @@ -67,3 +67,12 @@ func (memo *Memo) Get() (value interface{}, err error) { } return memo.cache.res.value, memo.cache.res.err } + +// Reset forcibly resets the cache to nil. +func (memo *Memo) Reset() { + defer memo.mu.Unlock() + memo.mu.Lock() + + memo.cache = nil + memo.lastCache = time.Now() +} From 7ddb288ae33e93666bad79e93c7bd0da9db5d791 Mon Sep 17 00:00:00 2001 From: Colin Axner Date: Sun, 6 Dec 2020 12:06:51 +0100 Subject: [PATCH 26/76] add godoc --- memo/memo.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/memo/memo.go b/memo/memo.go index 893bc79..8a88ccd 100644 --- a/memo/memo.go +++ b/memo/memo.go @@ -68,7 +68,8 @@ func (memo *Memo) Get() (value interface{}, err error) { return memo.cache.res.value, memo.cache.res.err } -// Reset forcibly resets the cache to nil. +// Reset forcibly resets the cache to nil without checking if the cache +// duration has elapsed. func (memo *Memo) Reset() { defer memo.mu.Unlock() memo.mu.Lock() From 48cbbf652a447eac5f55cb10ba27b713b6fd7387 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 24 Mar 2021 15:33:34 -0400 Subject: [PATCH 27/76] Support attachments on activitystreams.Object --- activitystreams/activity.go | 1 + activitystreams/attachment.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 activitystreams/attachment.go diff --git a/activitystreams/activity.go b/activitystreams/activity.go index 83f87b7..93a40af 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -117,6 +117,7 @@ type Object struct { Content string `json:"content"` ContentMap map[string]string `json:"contentMap,omitempty"` Tag []Tag `json:"tag"` + Attachment []Attachment `json:"attachment,omitempty"` // Extensions // NOTE: add extensions here diff --git a/activitystreams/attachment.go b/activitystreams/attachment.go new file mode 100644 index 0000000..b2b47e9 --- /dev/null +++ b/activitystreams/attachment.go @@ -0,0 +1,12 @@ +package activitystreams + +type Attachment struct { + Type AttachmentType `json:"type"` + URL string `json:"url"` + MediaType string `json:"mediaType"` + Name string `json:"name"` +} + +type AttachmentType string + +const AttachImage AttachmentType = "Image" From a77cafb3aef40a2eba9845254908ebd0aa151909 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 24 Mar 2021 15:45:58 -0400 Subject: [PATCH 28/76] Add NewImageAttachment func --- activitystreams/attachment.go | 21 ++++++++++++++++ activitystreams/attachment_test.go | 40 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 activitystreams/attachment_test.go diff --git a/activitystreams/attachment.go b/activitystreams/attachment.go index b2b47e9..c2bfc2e 100644 --- a/activitystreams/attachment.go +++ b/activitystreams/attachment.go @@ -1,5 +1,10 @@ package activitystreams +import ( + "mime" + "strings" +) + type Attachment struct { Type AttachmentType `json:"type"` URL string `json:"url"` @@ -10,3 +15,19 @@ type Attachment struct { type AttachmentType string const AttachImage AttachmentType = "Image" + +// NewImageAttachment creates a new Attachment from the given URL, setting the +// correct type and automatically detecting the MediaType based on the file +// extension. +func NewImageAttachment(url string) *Attachment { + var imgType string + extIdx := strings.LastIndexByte(url, '.') + if extIdx > -1 { + imgType = mime.TypeByExtension(url[extIdx:]) + } + return &Attachment{ + Type: AttachImage, + URL: url, + MediaType: imgType, + } +} diff --git a/activitystreams/attachment_test.go b/activitystreams/attachment_test.go new file mode 100644 index 0000000..c96d1cd --- /dev/null +++ b/activitystreams/attachment_test.go @@ -0,0 +1,40 @@ +package activitystreams + +import ( + "reflect" + "testing" +) + +func TestNewImageAttachment(t *testing.T) { + type args struct { + url string + } + tests := []struct { + name string + args args + want *Attachment + }{ + {name: "good svg", args: args{"https://writefreely.org/img/writefreely.svg"}, want: &Attachment{ + Type: "Image", + URL: "https://writefreely.org/img/writefreely.svg", + MediaType: "image/svg+xml", + }}, + {name: "good png", args: args{"https://i.snap.as/12345678.png"}, want: &Attachment{ + Type: "Image", + URL: "https://i.snap.as/12345678.png", + MediaType: "image/png", + }}, + {name: "no extension", args: args{"https://i.snap.as/12345678"}, want: &Attachment{ + Type: "Image", + URL: "https://i.snap.as/12345678", + MediaType: "", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewImageAttachment(tt.args.url); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewImageAttachment() = %v, want %v", got, tt.want) + } + }) + } +} From f534cc9f9b851c57f5189d212159e5b1d7f88343 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 24 Mar 2021 15:46:13 -0400 Subject: [PATCH 29/76] Fix TagMention type --- activitystreams/tag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitystreams/tag.go b/activitystreams/tag.go index 63fe4c4..38010bf 100644 --- a/activitystreams/tag.go +++ b/activitystreams/tag.go @@ -10,5 +10,5 @@ type TagType string const ( TagHashtag TagType = "Hashtag" - TagMention = "Mention" + TagMention TagType = "Mention" ) From 8dbd86535d0e5725384799f9ae9f6484c512146a Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 24 Mar 2021 15:47:23 -0400 Subject: [PATCH 30/76] Don't return pointer from NewImageAttachment() --- activitystreams/attachment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitystreams/attachment.go b/activitystreams/attachment.go index c2bfc2e..d221169 100644 --- a/activitystreams/attachment.go +++ b/activitystreams/attachment.go @@ -19,13 +19,13 @@ const AttachImage AttachmentType = "Image" // NewImageAttachment creates a new Attachment from the given URL, setting the // correct type and automatically detecting the MediaType based on the file // extension. -func NewImageAttachment(url string) *Attachment { +func NewImageAttachment(url string) Attachment { var imgType string extIdx := strings.LastIndexByte(url, '.') if extIdx > -1 { imgType = mime.TypeByExtension(url[extIdx:]) } - return &Attachment{ + return Attachment{ Type: AttachImage, URL: url, MediaType: imgType, From 95a3a717ed8f5ed245147a1b214218a8ad904098 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 30 Mar 2021 12:44:22 -0400 Subject: [PATCH 31/76] Move remaining writeas/nerds/store funcs to web-core Finishes the work started in #8. --- id/random.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/id/random.go b/id/random.go index 0c804df..222833d 100644 --- a/id/random.go +++ b/id/random.go @@ -1,8 +1,8 @@ package id import ( - "fmt" "crypto/rand" + "fmt" ) // GenerateRandomString creates a random string of characters of the given @@ -24,3 +24,15 @@ func GenerateRandomString(dictionary string, l int) string { func GenSafeUniqueSlug(slug string) string { return fmt.Sprintf("%s-%s", slug, GenerateRandomString("0123456789bcdfghjklmnpqrstvwxyz", 4)) } + +// Generate62RandomString creates a random string with the given length +// consisting of characters in [A-Za-z0-9]. +func Generate62RandomString(l int) string { + return GenerateRandomString("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", l) +} + +// GenerateFriendlyRandomString creates a random string of characters with the +// given length consisting of characters in [a-z0-9]. +func GenerateFriendlyRandomString(l int) string { + return GenerateRandomString("0123456789abcdefghijklmnopqrstuvwxyz", l) +} From fd1559928a0f11640eded30121574376c15a396e Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 13 Apr 2021 23:49:59 -0400 Subject: [PATCH 32/76] Copy post parsing funcs to posts pkg --- go.mod | 10 ++-- go.sum | 29 ++++++---- posts/parse.go | 144 ++++++++++++++++++++++++++++++++++++++++++++++++ posts/render.go | 7 +++ 4 files changed, 174 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index fe057de..0ea6e4d 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.10 require ( github.com/gofrs/uuid v3.3.0+incompatible - github.com/kr/pretty v0.1.0 // indirect github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec github.com/microcosm-cc/bluemonday v1.0.2 github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect - github.com/writeas/impart v1.1.0 + github.com/writeas/go-strip-markdown v2.0.1+incompatible + github.com/writeas/impart v1.1.1 github.com/writeas/openssl-go v1.0.0 - github.com/writeas/saturday v1.6.0 - golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + github.com/writeas/saturday v1.7.1 + golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect ) diff --git a/go.sum b/go.sum index 4d2be2e..81be546 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,7 @@ -github.com/gofrs/uuid v1.2.0 h1:coDhrjgyJaglxSjxuJdqQSSdUpG3w6p1OwN2od6frBU= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -12,17 +11,25 @@ github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+i github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/writeas/impart v1.1.0 h1:nPnoO211VscNkp/gnzir5UwCDEvdHThL5uELU60NFSE= -github.com/writeas/impart v1.1.0/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= +github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= +github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= +github.com/writeas/impart v1.1.1 h1:RyA9+CqbdbDuz53k+nXCWUY+NlEkdyw6+nWanxSBl5o= +github.com/writeas/impart v1.1.1/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjFc5o= github.com/writeas/openssl-go v1.0.0/go.mod h1:WsKeK5jYl0B5y8ggOmtVjbmb+3rEGqSD25TppjJnETA= -github.com/writeas/saturday v1.6.0 h1:HNUtX8TVJJnSdxE8vaAgtHiAJVt+Of5AJYlm62pmVlI= -github.com/writeas/saturday v1.6.0/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= -golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= -golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +github.com/writeas/saturday v1.7.1 h1:lYo1EH6CYyrFObQoA9RNWHVlpZA5iYL5Opxo7PYAnZE= +github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= diff --git a/posts/parse.go b/posts/parse.go index b082a05..168289f 100644 --- a/posts/parse.go +++ b/posts/parse.go @@ -1,7 +1,24 @@ package posts import ( + "fmt" + stripmd "github.com/writeas/go-strip-markdown" + "github.com/writeas/web-core/stringmanip" + "regexp" "strings" + "unicode" + "unicode/utf8" +) + +const ( + maxTitleLen = 80 + assumedTitleLen = 80 +) + +var ( + titleElementReg = regexp.MustCompile("") + urlReg = regexp.MustCompile("https?://") + imgReg = regexp.MustCompile(`!\[([^]]+)\]\([^)]+\)`) ) // ExtractTitle takes the given raw post text and returns a title, if explicitly @@ -19,3 +36,130 @@ func ExtractTitle(content string) (title string, body string) { body = content return } + +func FriendlyPostTitle(content, friendlyId string) string { + content = StripHTMLWithoutEscaping(content) + + content = strings.TrimLeftFunc(stripmd.Strip(content), unicode.IsSpace) + eol := strings.IndexRune(content, '\n') + blankLine := strings.Index(content, "\n\n") + if blankLine != -1 && blankLine <= eol && blankLine <= assumedTitleLen { + return strings.TrimSpace(content[:blankLine]) + } else if eol == -1 && utf8.RuneCountInString(content) <= maxTitleLen { + return content + } + + title, truncd := TruncToWord(PostLede(content, true), maxTitleLen) + if truncd { + title += "..." + } + return title +} + +// PostDescription generates a description based on the given post content, +// title, and post ID. This doesn't consider a V2 post field, `title` when +// choosing what to generate. In case a post has a title, this function will +// fail, and logic should instead be implemented to skip this when there's no +// title, like so: +// var desc string +// if title == "" { +// desc = PostDescription(content, title, friendlyId) +// } else { +// desc = ShortPostDescription(content) +// } +func PostDescription(content, title, friendlyId string) string { + maxLen := 140 + + if content == "" { + content = "WriteFreely is a painless, simple, federated blogging platform." + } else { + fmtStr := "%s" + truncation := 0 + if utf8.RuneCountInString(content) > maxLen { + // Post is longer than the max description, so let's show a better description + fmtStr = "%s..." + truncation = 3 + } + + if title == friendlyId { + // No specific title was found; simply truncate the post, starting at the beginning + content = fmt.Sprintf(fmtStr, strings.Replace(stringmanip.Substring(content, 0, maxLen-truncation), "\n", " ", -1)) + } else { + // There was a title, so return a real description + blankLine := strings.Index(content, "\n\n") + if blankLine < 0 { + blankLine = 0 + } + truncd := stringmanip.Substring(content, blankLine, blankLine+maxLen-truncation) + contentNoNL := strings.Replace(truncd, "\n", " ", -1) + content = strings.TrimSpace(fmt.Sprintf(fmtStr, contentNoNL)) + } + } + + return content +} + +func ShortPostDescription(content string) string { + maxLen := 140 + fmtStr := "%s" + truncation := 0 + if utf8.RuneCountInString(content) > maxLen { + // Post is longer than the max description, so let's show a better description + fmtStr = "%s..." + truncation = 3 + } + return strings.TrimSpace(fmt.Sprintf(fmtStr, strings.Replace(stringmanip.Substring(content, 0, maxLen-truncation), "\n", " ", -1))) +} + +// TruncToWord truncates the given text to the provided limit. +func TruncToWord(s string, l int) (string, bool) { + truncated := false + c := []rune(s) + if len(c) > l { + truncated = true + s = string(c[:l]) + spaceIdx := strings.LastIndexByte(s, ' ') + if spaceIdx > -1 { + s = s[:spaceIdx] + } + } + return s, truncated +} + +// PostLede attempts to extract the first thought of the given post, generally +// contained within the first line or sentence of text. +func PostLede(t string, includePunc bool) string { + // Adjust where we truncate if we want to include punctuation + iAdj := 0 + if includePunc { + iAdj = 1 + } + + // Find lede within first line of text + nl := strings.IndexRune(t, '\n') + if nl > -1 { + t = t[:nl] + } + + // Strip certain HTML tags + t = titleElementReg.ReplaceAllString(t, "") + + // Strip URL protocols + t = urlReg.ReplaceAllString(t, "") + + // Strip image URL, leaving only alt text + t = imgReg.ReplaceAllString(t, " $1 ") + + // Find lede within first sentence + punc := strings.Index(t, ". ") + if punc > -1 { + t = t[:punc+iAdj] + } + punc = stringmanip.IndexRune(t, '。') + if punc > -1 { + c := []rune(t) + t = string(c[:punc+iAdj]) + } + + return t +} diff --git a/posts/render.go b/posts/render.go index 157bed2..d0885d5 100644 --- a/posts/render.go +++ b/posts/render.go @@ -3,6 +3,7 @@ package posts import ( "github.com/microcosm-cc/bluemonday" "github.com/writeas/saturday" + "html" "regexp" "strings" "unicode" @@ -61,3 +62,9 @@ func ApplyBasicMarkdown(data []byte) string { return outHTML } + +// StripHTMLWithoutEscaping strips HTML tags with bluemonday's StrictPolicy, then unescapes the HTML +// entities added in by sanitizing the content. +func StripHTMLWithoutEscaping(content string) string { + return html.UnescapeString(bluemonday.StrictPolicy().Sanitize(content)) +} From 5fb147fe78b2a987bf2dd629dd87fb48969fac31 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 21 Apr 2021 11:50:00 -0400 Subject: [PATCH 33/76] Add PostLede and TruncToWord tests from WriteFreely --- posts/parse_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/posts/parse_test.go b/posts/parse_test.go index 71245fe..c2a8066 100644 --- a/posts/parse_test.go +++ b/posts/parse_test.go @@ -33,3 +33,45 @@ Yep.`}, } } } + +func TestPostLede(t *testing.T) { + text := map[string]string{ + "早安。跨出舒適圈，才能前往": "早安。", + "早安。This is my post. It is great.": "早安。", + "Hello. 早安。": "Hello.", + "Sup? Everyone says punctuation is punctuation.": "Sup?", + "Humans are humans, and society is full of good and bad actors. Technology, at the most fundamental level, is a neutral tool that can be used by either to meet any ends. ": "Humans are humans, and society is full of good and bad actors.", + `Online Domino Is Must For Everyone + + Do you want to understand how to play poker online?`: "Online Domino Is Must For Everyone", + `おはようございます + + 私は日本から帰ったばかりです。`: "おはようございます", + "Hello, we say, おはよう. We say \"good morning\"": "Hello, we say, おはよう.", + } + + c := 1 + for i, o := range text { + if s := PostLede(i, true); s != o { + t.Errorf("#%d: Got '%s' from '%s'; expected '%s'", c, s, i, o) + } + c++ + } +} + +func TestTruncToWord(t *testing.T) { + text := map[string]string{ + "Можливо, ми можемо використовувати інтернет-інструменти, щоб виготовити якийсь текст, який би міг бути і на, і в кінцевому підсумку, буде скорочено, тому що це тривало так довго.": "Можливо, ми можемо використовувати інтернет-інструменти, щоб виготовити якийсь", + "早安。This is my post. It is great. It is a long post that is great that is a post that is great.": "早安。This is my post. It is great. It is a long post that is great that is a post", + "Sup? Everyone says punctuation is punctuation.": "Sup? Everyone says punctuation is punctuation.", + "I arrived in Japan six days ago. Tired from a 10-hour flight after a night-long layover in Calgary, I wandered wide-eyed around Narita airport looking for an ATM.": "I arrived in Japan six days ago. Tired from a 10-hour flight after a night-long", + } + + c := 1 + for i, o := range text { + if s, _ := TruncToWord(i, 80); s != o { + t.Errorf("#%d: Got '%s' from '%s'; expected '%s'", c, s, i, o) + } + c++ + } +} From 8b45bd4fcc845271717b951a682c1fbbf19e84ab Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 13 Sep 2021 15:32:10 -0400 Subject: [PATCH 34/76] Support creating Document attachments In particular, this will support audio attachments. Ref T872 --- activitystreams/attachment.go | 24 +++++++++++++++++++----- activitystreams/attachment_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/activitystreams/attachment.go b/activitystreams/attachment.go index d221169..f8763e3 100644 --- a/activitystreams/attachment.go +++ b/activitystreams/attachment.go @@ -14,20 +14,34 @@ type Attachment struct { type AttachmentType string -const AttachImage AttachmentType = "Image" +const ( + AttachImage AttachmentType = "Image" + AttachDocument AttachmentType = "Document" +) // NewImageAttachment creates a new Attachment from the given URL, setting the // correct type and automatically detecting the MediaType based on the file // extension. func NewImageAttachment(url string) Attachment { - var imgType string + return newAttachment(url, AttachImage) +} + +// NewDocumentAttachment creates a new Attachment from the given URL, setting the +// correct type and automatically detecting the MediaType based on the file +// extension. +func NewDocumentAttachment(url string) Attachment { + return newAttachment(url, AttachDocument) +} + +func newAttachment(url string, attachType AttachmentType) Attachment { + var fileType string extIdx := strings.LastIndexByte(url, '.') if extIdx > -1 { - imgType = mime.TypeByExtension(url[extIdx:]) + fileType = mime.TypeByExtension(url[extIdx:]) } return Attachment{ - Type: AttachImage, + Type: attachType, URL: url, - MediaType: imgType, + MediaType: fileType, } } diff --git a/activitystreams/attachment_test.go b/activitystreams/attachment_test.go index c96d1cd..e0d3b4d 100644 --- a/activitystreams/attachment_test.go +++ b/activitystreams/attachment_test.go @@ -38,3 +38,27 @@ func TestNewImageAttachment(t *testing.T) { }) } } + +func TestNewDocumentAttachment(t *testing.T) { + type args struct { + url string + } + tests := []struct { + name string + args args + want Attachment + }{ + {name: "mp3", args: args{"https://listen.as/matt/abc.mp3"}, want: Attachment{ + Type: "Document", + URL: "https://listen.as/matt/abc.mp3", + MediaType: "audio/mpeg", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewDocumentAttachment(tt.args.url); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewDocumentAttachment() = %+v, want %+v", got, tt.want) + } + }) + } +} From 28cf6b3866b0f1526748a25173e3ace61b6da33e Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 13 Sep 2021 15:32:27 -0400 Subject: [PATCH 35/76] Fix TestNewImageAttachment tests --- activitystreams/attachment_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/activitystreams/attachment_test.go b/activitystreams/attachment_test.go index e0d3b4d..76a524b 100644 --- a/activitystreams/attachment_test.go +++ b/activitystreams/attachment_test.go @@ -12,19 +12,19 @@ func TestNewImageAttachment(t *testing.T) { tests := []struct { name string args args - want *Attachment + want Attachment }{ - {name: "good svg", args: args{"https://writefreely.org/img/writefreely.svg"}, want: &Attachment{ + {name: "good svg", args: args{"https://writefreely.org/img/writefreely.svg"}, want: Attachment{ Type: "Image", URL: "https://writefreely.org/img/writefreely.svg", MediaType: "image/svg+xml", }}, - {name: "good png", args: args{"https://i.snap.as/12345678.png"}, want: &Attachment{ + {name: "good png", args: args{"https://i.snap.as/12345678.png"}, want: Attachment{ Type: "Image", URL: "https://i.snap.as/12345678.png", MediaType: "image/png", }}, - {name: "no extension", args: args{"https://i.snap.as/12345678"}, want: &Attachment{ + {name: "no extension", args: args{"https://i.snap.as/12345678"}, want: Attachment{ Type: "Image", URL: "https://i.snap.as/12345678", MediaType: "", From bb86406b9d092b90e2ad75b28a07c11fbe065cda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 19:34:04 +0000 Subject: [PATCH 36/76] Bump github.com/microcosm-cc/bluemonday from 1.0.2 to 1.0.5 Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.2 to 1.0.5. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.2...v1.0.5) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 0ea6e4d..a8769ef 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.10 require ( github.com/gofrs/uuid v3.3.0+incompatible github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec - github.com/microcosm-cc/bluemonday v1.0.2 + github.com/microcosm-cc/bluemonday v1.0.5 github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/writeas/go-strip-markdown v2.0.1+incompatible github.com/writeas/impart v1.1.1 diff --git a/go.sum b/go.sum index 81be546..9780f16 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU= +github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -7,8 +13,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec h1:ZXWuspqypleMuJy4bzYEqlMhJnGAYpLrWe5p7W3CdvI= github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec/go.mod h1:voECJzdraJmolzPBgL9Z7ANwXf4oMXaTCsIkdiPpR/g= -github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/microcosm-cc/bluemonday v1.0.5 h1:cF59UCKMmmUgqN1baLvqU/B1ZsMori+duLVTLpgiG3w= +github.com/microcosm-cc/bluemonday v1.0.5/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= @@ -22,7 +28,6 @@ github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= From a8daef840104f953f1a5a5fd4a51e77c6a813cd9 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 29 Sep 2021 17:01:17 -0400 Subject: [PATCH 37/76] Add category pkg This supports categories in Write.as / WriteFreely. Ref T809 --- category/category.go | 30 ++++++++++++++++++++++++++++++ category/tags.go | 26 ++++++++++++++++++++++++++ category/tags_test.go | 31 +++++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 4 ++++ 5 files changed, 93 insertions(+) create mode 100644 category/category.go create mode 100644 category/tags.go create mode 100644 category/tags_test.go diff --git a/category/category.go b/category/category.go new file mode 100644 index 0000000..f60dc4b --- /dev/null +++ b/category/category.go @@ -0,0 +1,30 @@ +// Package category supports post categories +package category + +import ( + "errors" + "github.com/writeas/slug" +) + +var ( + ErrNotFound = errors.New("category doesn't exist") +) + +// Category represents a post tag with additional metadata, like a title and slug. +type Category struct { + ID int64 `json:"-"` + Hashtag string `json:"hashtag"` + Slug string `json:"slug"` + Title string `json:"title"` +} + +// NewCategory creates a Category you can insert into the database, based on a hashtag. It automatically breaks up the +// hashtag by words, based on capitalization, for both the title and a URL-friendly slug. +func NewCategory(hashtag string) *Category { + title := titleFromHashtag(hashtag) + return &Category{ + Hashtag: hashtag, + Slug: slug.Make(title), + Title: title, + } +} diff --git a/category/tags.go b/category/tags.go new file mode 100644 index 0000000..9b99248 --- /dev/null +++ b/category/tags.go @@ -0,0 +1,26 @@ +package category + +import ( + "strings" + "unicode" +) + +// titleFromHashtag generates an all-lowercase title, with spaces inserted based on initial capitalization -- e.g. +// "MyWordyTag" becomes "my wordy tag". +func titleFromHashtag(hashtag string) string { + var t strings.Builder + var prev rune + for i, c := range hashtag { + if unicode.IsUpper(c) { + if i > 0 && !unicode.IsUpper(prev) { + // Insert space if previous rune wasn't also uppercase (e.g. an abbreviation) + t.WriteRune(' ') + } + t.WriteRune(unicode.ToLower(c)) + } else { + t.WriteRune(c) + } + prev = c + } + return t.String() +} diff --git a/category/tags_test.go b/category/tags_test.go new file mode 100644 index 0000000..9889cfc --- /dev/null +++ b/category/tags_test.go @@ -0,0 +1,31 @@ +package category + +import "testing" + +func TestTitleFromHashtag(t *testing.T) { + tests := []struct { + name string + hashtag string + expTitle string + }{ + {"proper noun", "Jane", "jane"}, + {"full name", "JaneDoe", "jane doe"}, + {"us words", "unitedStates", "united states"}, + {"usa", "USA", "usa"}, + {"us monoword", "unitedstates", "unitedstates"}, + {"100dto", "100DaysToOffload", "100 days to offload"}, + {"iphone", "iPhone", "iphone"}, + {"ilike", "iLikeThis", "i like this"}, + {"abird", "aBird", "a bird"}, + {"all caps", "URGENT", "urgent"}, + {"smartphone", "スマートフォン", "スマートフォン"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res := titleFromHashtag(test.hashtag) + if res != test.expTitle { + t.Fatalf("#%s: got '%s' expected '%s'", test.hashtag, res, test.expTitle) + } + }) + } +} diff --git a/go.mod b/go.mod index a8769ef..5170100 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,13 @@ require ( github.com/gofrs/uuid v3.3.0+incompatible github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec github.com/microcosm-cc/bluemonday v1.0.5 + github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/writeas/go-strip-markdown v2.0.1+incompatible github.com/writeas/impart v1.1.1 github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.7.1 + github.com/writeas/slug v1.2.0 golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect diff --git a/go.sum b/go.sum index 9780f16..e02806f 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec h1:ZXWuspq github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec/go.mod h1:voECJzdraJmolzPBgL9Z7ANwXf4oMXaTCsIkdiPpR/g= github.com/microcosm-cc/bluemonday v1.0.5 h1:cF59UCKMmmUgqN1baLvqU/B1ZsMori+duLVTLpgiG3w= github.com/microcosm-cc/bluemonday v1.0.5/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w= +github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ= +github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= @@ -25,6 +27,8 @@ github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjF github.com/writeas/openssl-go v1.0.0/go.mod h1:WsKeK5jYl0B5y8ggOmtVjbmb+3rEGqSD25TppjJnETA= github.com/writeas/saturday v1.7.1 h1:lYo1EH6CYyrFObQoA9RNWHVlpZA5iYL5Opxo7PYAnZE= github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= +github.com/writeas/slug v1.2.0 h1:EMQ+cwLiOcA6EtFwUgyw3Ge18x9uflUnOnR6bp/J+/g= +github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQFubfQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= From 375d22d23009e361ef9e0cf9e0d1ab62e13a25a9 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 16 Oct 2021 13:19:00 -0400 Subject: [PATCH 38/76] Add HashtagFromTitle helper --- category/tags.go | 21 +++++++++++++++++++++ category/tags_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/category/tags.go b/category/tags.go index 9b99248..a3a092a 100644 --- a/category/tags.go +++ b/category/tags.go @@ -24,3 +24,24 @@ func titleFromHashtag(hashtag string) string { } return t.String() } + +// HashtagFromTitle generates a valid single-word, camelCase hashtag from a title (which might include spaces, +// punctuation, etc.). +func HashtagFromTitle(title string) string { + var t strings.Builder + var prev rune + for _, c := range title { + if !unicode.IsLetter(c) && !unicode.IsNumber(c) { + prev = c + continue + } + if unicode.IsSpace(prev) { + // Uppercase next word + t.WriteRune(unicode.ToUpper(c)) + } else { + t.WriteRune(c) + } + prev = c + } + return t.String() +} diff --git a/category/tags_test.go b/category/tags_test.go index 9889cfc..92a45c1 100644 --- a/category/tags_test.go +++ b/category/tags_test.go @@ -29,3 +29,32 @@ func TestTitleFromHashtag(t *testing.T) { }) } } + +func TestHashtagFromTitle(t *testing.T) { + tests := []struct { + name string + title string + expHashtag string + }{ + {"proper noun", "Jane", "Jane"}, + {"full name", "Jane Doe", "JaneDoe"}, + {"us upper words", "United States", "UnitedStates"}, + {"us lower words", "united states", "unitedStates"}, + {"usa", "USA", "USA"}, + {"100dto", "100 Days To Offload", "100DaysToOffload"}, + {"iphone", "iPhone", "iPhone"}, + {"ilike", "I like this", "ILikeThis"}, + {"abird", "a Bird", "aBird"}, + {"all caps", "URGENT", "URGENT"}, + {"punctuation", "John’s Stories", "JohnsStories"}, + {"smartphone", "スマートフォン", "スマートフォン"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res := HashtagFromTitle(test.title) + if res != test.expHashtag { + t.Fatalf("%s: got '%s' expected '%s'", test.title, res, test.expHashtag) + } + }) + } +} From f276f4d64a2320f0e2024c1a840a62fbf4613811 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 16 Oct 2021 15:27:37 -0400 Subject: [PATCH 39/76] Add NewCategoryFromPartial func --- category/category.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/category/category.go b/category/category.go index f60dc4b..bc427b1 100644 --- a/category/category.go +++ b/category/category.go @@ -28,3 +28,24 @@ func NewCategory(hashtag string) *Category { Title: title, } } + +// NewCategoryFromPartial creates a Category from a partially-populated Category, such as when a user initially creates +// one. +func NewCategoryFromPartial(cat *Category) *Category { + newCat := &Category{ + Hashtag: cat.Hashtag, + } + // Create title from hashtag, if none supplied + if cat.Title == "" { + newCat.Title = titleFromHashtag(cat.Hashtag) + } else { + newCat.Title = cat.Title + } + // Create slug from title, if none supplied; otherwise ensure slug is valid + if cat.Slug == "" { + newCat.Slug = slug.Make(newCat.Title) + } else { + newCat.Slug = slug.Make(cat.Slug) + } + return newCat +} From ba8581960724242e010746ee3d1c50fec75c7ab2 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 3 Nov 2021 13:59:21 -0400 Subject: [PATCH 40/76] Add PostCount property to Category --- category/category.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/category/category.go b/category/category.go index bc427b1..4ac81c3 100644 --- a/category/category.go +++ b/category/category.go @@ -12,10 +12,11 @@ var ( // Category represents a post tag with additional metadata, like a title and slug. type Category struct { - ID int64 `json:"-"` - Hashtag string `json:"hashtag"` - Slug string `json:"slug"` - Title string `json:"title"` + ID int64 `json:"-"` + Hashtag string `json:"hashtag"` + Slug string `json:"slug"` + Title string `json:"title"` + PostCount int64 `json:"post_count"` } // NewCategory creates a Category you can insert into the database, based on a hashtag. It automatically breaks up the From c643a95bdaa5b6d2f70dff94f2e43f2f415b31c0 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:30:08 -0400 Subject: [PATCH 41/76] Update French translations --- l10n/phrases_fr.go | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/l10n/phrases_fr.go b/l10n/phrases_fr.go index fc3af32..ce1603a 100644 --- a/l10n/phrases_fr.go +++ b/l10n/phrases_fr.go @@ -1,24 +1,28 @@ package l10n var phrasesFR = map[string]string{ - "Anonymous post": "Post anonyme", - "Blogs": "Blogs", - "Newer": "Nouveaux", - "Older": "Anciens", - "Posts": "Posts", - "Publish to...": "Publie sur...", - "Publish": "Publie", - "Read more...": "En savoir plus...", - "Toggle theme": "Activer thème", - "View posts": "Voir Posts", - "delete": "effacer", - "edit": "modifier", - "move to...": "déplacer vers...", - "pin": "épingler", - "published with write.as": "publié avec write.as", - "share modal ending": "Envoie-le à un ami, partage-le sur le web, ou peut-être en tant que tweet. En savoir plus.", - "share modal introduction": "Tout post publié a une adresse URL secrète et unique que tu peux partager avec qui tu veux. Voici cette URL:", - "share modal title": "Partage ce post", - "share": "partager", - "unpin": "enlever", + "Anonymous post": "Billet anonyme", + "Blogs": "Blogs", + "Enter": "Valider", + "Newer": "Récents", + "Older": "Précédents", + "Posts": "Billets", + "Publish to...": "Publier sur...", + "Publish": "Publier", + "Read more...": "Lire la suite...", + "Subscribe": "S'abonner", + "This blog requires a password.": "Ce blog requiert un mot de passe.", + "Toggle theme": "Changer de thème", + "View posts": "Voir les billets", + "delete": "effacer", + "edit": "modifier", + "email subscription prompt": "Insérer votre adresse email pour recevoir les mises à jour", + "move to...": "déplacer vers...", + "pin": "épingler", + "published with write.as": "publié avec write.as", + "share modal ending": "Envoyer à un ami, partager sur le web, ou peut-être en tant que tweet. En savoir plus.", + "share modal introduction": "Chaque billet dispose d’une adresse (URL) secrète et unique qui peut être partagée avec quelqu’un. Voici cette URL:", + "share modal title": "Partager ce billet", + "share": "partager", + "unpin": "détacher", } From 982e31c5bec47d4b767f3e618d44bc4c3e92a1c5 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:32:33 -0400 Subject: [PATCH 42/76] Update Arabic translation --- l10n/phrases_ar.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/l10n/phrases_ar.go b/l10n/phrases_ar.go index 70d83f5..8dbc86d 100644 --- a/l10n/phrases_ar.go +++ b/l10n/phrases_ar.go @@ -10,17 +10,19 @@ var phrasesAR = map[string]string{ "Publish to...": "النشر إلى…", "Publish": "نشر", "Read more...": "اقرأ المزيد…", + "Subscribe": "اشترك", "This blog requires a password.": "هذه المدونة تتطلب کلمة سرية.", "Toggle theme": "تغيير القالب", "View posts": "مشاهدة المنشورات", "delete": "حذف", "edit": "تعديل", + "email subscription prompt": "ادخل عنوان بريدك الإلكتروني للإشتراك في التحديثات", "move to...": "أنقله إلى…", "pin": "تدبيس", - "published with write.as": "مدعوم من write.as", - "share modal ending": "ارسلهه إلى صديق، شاركه عبر الويب، أو غرّده. تعلّم المزيد.", - "share modal introduction": "كل الفيديوهات المنشورة لديها رابط سري مميز، يمكنك مشاركته. هذا هو الرابط:", - "share modal title": "شارك هذا المنشور", - "share": "شارك", - "unpin": "إلغاء التدبيس", + "published with write.as": "مدعوم من write.as", + "share modal ending": "ارسلهه إلى صديق، شاركه عبر الويب، أو غرّده. تعلّم المزيد.", + "share modal introduction": "كل الفيديوهات المنشورة لديها رابط سري مميز، يمكنك مشاركته. هذا هو الرابط:", + "share modal title": "شارك هذا المنشور", + "share": "شارك", + "unpin": "إلغاء التدبيس", } From 44e795b77db59f98d7902acef742f0dbd1ed6e19 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:38:14 -0400 Subject: [PATCH 43/76] Update Chinese translation --- l10n/phrases_zh.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/l10n/phrases_zh.go b/l10n/phrases_zh.go index 4a4ec25..454c555 100644 --- a/l10n/phrases_zh.go +++ b/l10n/phrases_zh.go @@ -3,24 +3,26 @@ package l10n var phrasesZH = map[string]string{ "Anonymous post": "匿名文章", "Blogs": "博客", - "Enter": "按下回车", - "Newer": "更新", - "Older": "较旧", + "Enter": "进入", + "Newer": "最近的博客", + "Older": "之前的博客", "Posts": "文章", "Publish to...": "发布到...", "Publish": "发布", "Read more...": "阅读更多", - "This blog requires a password.": "这篇文章需要密码", + "Subscribe": "订阅", + "This blog requires a password.": "这篇博客需要密码", "Toggle theme": "更换主题", "View posts": "查看文章", "delete": "删除", "edit": "编辑", + "email subscription prompt": "输入邮件地址，订阅更新", "move to...": "移动到", "pin": "固定文章", - "published with write.as": "通过 write.as 发布", - "share modal ending": "发送给朋友、与世界分享或者发条推。了解更多。", - "share modal introduction": "发布的每篇文章都有一个唯一的私有网址，你可以把它分享给任何人：", - "share modal title": "分享文章", - "share": "分享", - "unpin": "取消固定", + "published with write.as": "用write.as来发布", + "share modal ending": "发送给朋友，或者线上分享，如果可能的话，分享到推特上。了解更多。", + "share modal introduction": "发布的每篇文章都有一个唯一的私有URL，你可以把它分享给任何人。这是URL：", + "share modal title": "分享这篇文章", + "share": "分享", + "unpin": "取消固定", } From 3b7d4deeaa8783b523636481c2c75747b28e8ac2 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:41:30 -0400 Subject: [PATCH 44/76] Add Czech translations --- l10n/phrases_cs.go | 28 ++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 l10n/phrases_cs.go diff --git a/l10n/phrases_cs.go b/l10n/phrases_cs.go new file mode 100644 index 0000000..bc9cacf --- /dev/null +++ b/l10n/phrases_cs.go @@ -0,0 +1,28 @@ +package l10n + +var phrasesCS = map[string]string{ + "Anonymous post": "Anonymní článek", + "Blogs": "Blogy", + "Enter": "Vstoupit", + "Newer": "Novější", + "Older": "Starší", + "Posts": "Články", + "Publish to...": "Zveřejnit do...", + "Publish": "Zveřejnit", + "Read more...": "Číst dále...", + "Subscribe": "Odebírat", + "This blog requires a password.": "Tento blog je zaheslován.", + "Toggle theme": "Přepnout vzhled", + "View posts": "Zobrazit články", + "delete": "smazat", + "edit": "upravit", + "email subscription prompt": "Zadejte svůj e-mail a přihlaste se k odběru aktualizací.", + "move to...": "přesunout do...", + "pin": "připnout", + "published with write.as": "publikováno pomocí write.as", + "share modal ending": "Pošlete ji kamarádům, sdílejte na webu nebo na sociálních sítích. Zjistit více.", + "share modal introduction": "Každý zveřejněný článek má tajnou unikátní adresu, kterou můžete komukoliv poslat. Unikátní adresa pro tento článek je:", + "share modal title": "Sdílet tento článek", + "share": "sdílet", + "unpin": "odepnout", +} diff --git a/l10n/strings.go b/l10n/strings.go index edf9eb0..0825325 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -6,6 +6,8 @@ func Strings(lang string) map[string]string { switch lang { case "ar": return phrasesAR + case "cs": + return phrasesCS case "de": return phrasesDE case "el": From 26a278eba8474557398655593e323631e20f1d8a Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:43:35 -0400 Subject: [PATCH 45/76] Add new phrases (defaults) --- l10n/phrases.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/l10n/phrases.go b/l10n/phrases.go index 1040635..b1c10cb 100644 --- a/l10n/phrases.go +++ b/l10n/phrases.go @@ -14,23 +14,28 @@ var phrases = map[string]string{ "Blogs": "Blogs", "Enter": "Enter", "Newer": "Newer", + "Next": "Next", "Older": "Older", "Posts": "Posts", + "Previous": "Previous", "Publish to...": "Publish to...", "Publish": "Publish", "Read more...": "Read more...", + "Subscribe": "Subscribe", "This blog requires a password.": "This blog requires a password.", "Toggle theme": "Toggle theme", "View posts": "View Posts", "delete": "delete", "edit": "edit", + "email subscription confirm": "Please check your email and click the confirmation link to subscribe.", + "email subscription prompt": "Enter your email to subscribe to updates.", + "email subscription success": "Subscribed. You'll now receive future blog posts via email.", "move to...": "move to...", "pin": "pin", - "published with write.as": "published with write.as", - "share modal ending": "Send it to a friend, share it across the web, or maybe tweet it. Learn more.", - "share modal introduction": "Each published post has a secret, unique URL you can share with anyone. This is that URL:", - "share modal title": "Share this post", - "share": "share", - "unpin": "unpin", - "title dash": "—", + "published with write.as": "published with write.as", + "share modal ending": "Send it to a friend, share it across the web, or maybe tweet it. Learn more.", + "share modal introduction": "Each published post has a secret, unique URL you can share with anyone. This is that URL:", + "share modal title": "Share this post", + "share": "share", + "unpin": "unpin", } From 11821b1fe1d42a826a125073f165dfa305e5f784 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:45:30 -0400 Subject: [PATCH 46/76] Add Danish translations --- l10n/phrases_da.go | 28 ++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 l10n/phrases_da.go diff --git a/l10n/phrases_da.go b/l10n/phrases_da.go new file mode 100644 index 0000000..67f5fc3 --- /dev/null +++ b/l10n/phrases_da.go @@ -0,0 +1,28 @@ +package l10n + +var phrasesDA = map[string]string{ + "Anonymous post": "Anonymt indlæg", + "Blogs": "Blogs", + "Enter": "Enter", + "Newer": "Nyere", + "Older": "Ældre", + "Posts": "Indlæg", + "Publish to...": "Udgiv på...", + "Publish": "Udgiv", + "Read more...": "Læs videre...", + "Subscribe": "Abonnér", + "This blog requires a password.": "Den her blog kræver en adgangskode.", + "Toggle theme": "Skift design", + "View posts": "Vis indlæg", + "delete": "slet", + "edit": "redigér", + "email subscription prompt": "Indtast din email for at få tilsendt opdateringer.", + "move to...": "flyt til...", + "pin": "sæt øverst", + "published with write.as": "udgivet med write.as", + "share modal ending": "Send det til en ven, del det på nettet, eller lav et tweet. Læs mere.", + "share modal introduction": "Hvert indlæg har sit eget, hemmelige link, som du kan dele med enhver. Her er det førnævnte link:", + "share modal title": "Del indlæg", + "share": "del", + "unpin": "fjern øverst", +} diff --git a/l10n/strings.go b/l10n/strings.go index 0825325..19eede5 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -8,6 +8,8 @@ func Strings(lang string) map[string]string { return phrasesAR case "cs": return phrasesCS + case "da": + return phrasesDA case "de": return phrasesDE case "el": From 05ae34b39949b00e4b7b6c479ad9b04e1324e6c3 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:47:36 -0400 Subject: [PATCH 47/76] Add Dutch translations --- l10n/phrases_nl.go | 28 ++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 l10n/phrases_nl.go diff --git a/l10n/phrases_nl.go b/l10n/phrases_nl.go new file mode 100644 index 0000000..b5f7941 --- /dev/null +++ b/l10n/phrases_nl.go @@ -0,0 +1,28 @@ +package l10n + +var phrasesNL = map[string]string{ + "Anonymous post": "Anoniem artikel", + "Blogs": "Blogs", + "Enter": "Invoeren", + "Newer": "Nieuwer", + "Older": "Ouder", + "Posts": "Artikelen", + "Publish to...": "Publiceren op...", + "Publish": "Publiceren", + "Read more...": "Lees verder...", + "Subscribe": "Abonneren", + "This blog requires a password.": "Voor dit blog is een wachtwoord vereist.", + "Toggle theme": "Ander thema", + "View posts": "Artikelen bekijken", + "delete": "verwijderen", + "edit": "bewerken", + "email subscription prompt": "Voer je e-mailadres in om te abonneren op updates.", + "move to...": "verplaatsen naar...", + "pin": "vastmaken", + "published with write.as": "gepubliceerd met write.as", + "share modal ending": "Deel de link met een vriend, op het internet of op Twitter. Meer informatie.", + "share modal introduction": "Elk gepubliceerd artikel bevat een geheime, unieke link die je met iedereen kunt delen. Dit is de link:", + "share modal title": "Deel dit artikel", + "share": "delen", + "unpin": "losmaken", +} diff --git a/l10n/strings.go b/l10n/strings.go index 19eede5..b31d6a8 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -32,6 +32,8 @@ func Strings(lang string) map[string]string { return phrasesLT case "mk": return phrasesMK + case "nl": + return phrasesNL case "pl": return phrasesPL case "pt": From 61c88f6532499aade368664e83f25a28768ab41b Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:49:53 -0400 Subject: [PATCH 48/76] Update Esperanto translation --- l10n/phrases_eo.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/l10n/phrases_eo.go b/l10n/phrases_eo.go index 9086f3e..306ce40 100644 --- a/l10n/phrases_eo.go +++ b/l10n/phrases_eo.go @@ -17,10 +17,10 @@ var phrasesEO = map[string]string{ "edit": "redakti", "move to...": "movi al...", "pin": "alpingli", - "published with write.as": "Eldonita per write.as", - "share modal ending": "Sendu ĝin al unu amiko, kunhavu ĝin tra la retejo, aŭ eble tweet ĝin. Lernu pli.", - "share modal introduction": "Ĉiu eldonita afiŝo havas sekretan, unikan URL-on, kiun vi povas kunhavigi kun iu ajn. Ĉi tiu estas tiu URL-o:", - "share modal title": "Kunhavigi ĉi tiun afiŝon", - "share": "kunhavigi", - "unpin": "malalpingli", + "published with write.as": "Eldonita per write.as", + "share modal ending": "Sendu ĝin al unu amiko, kunhavu ĝin tra la retejo, aŭ eble tweet ĝin. Lernu pli.", + "share modal introduction": "Ĉiu eldonita afiŝo havas sekretan, unikan URL-on, kiun vi povas kunhavigi kun iu ajn. Ĉi tiu estas tiu URL-o:", + "share modal title": "Kunhavigi ĉi tiun afiŝon", + "share": "kunhavigi", + "unpin": "malalpingli", } From ed63d3d637301e7febea27bab7b0fee612805f90 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:51:06 -0400 Subject: [PATCH 49/76] Add Basque translations --- l10n/phrases_eu.go | 28 ++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 l10n/phrases_eu.go diff --git a/l10n/phrases_eu.go b/l10n/phrases_eu.go new file mode 100644 index 0000000..a840ba2 --- /dev/null +++ b/l10n/phrases_eu.go @@ -0,0 +1,28 @@ +package l10n + +var phrasesEU = map[string]string{ + "Anonymous post": "Bidalketa anonimoa", + "Blogs": "Blogak", + "Enter": "Sartu", + "Newer": "Berriagoak", + "Older": "Zaharragoak", + "Posts": "Bidalketak", + "Publish to...": "Argitaratu hemen...", + "Publish": "Argitaratu", + "Read more...": "Irakurri gehiago...", + "Subscribe": "Izena eman", + "This blog requires a password.": "Blog honek pasahitza behar du.", + "Toggle theme": "Aldatu itxura", + "View posts": "Ikusi bidalketak", + "delete": "ezabatu", + "edit": "editatu", + "email subscription prompt": "Sartu zure e-posta helbidea berriak jasotzeko.", + "move to...": "mugitu hona...", + "pin": "finkatu", + "published with write.as": "write.as bidez argitaratua", + "share modal ending": "Bidali lagun bati, partekatu sarean zehar edo agian, txiokatu. Ikasi gehiago.", + "share modal introduction": "Argitaratutako bidalketa orok URL sekretu eta bakarra dauka, edonorekin partekatu dezakezuna. Hau da URLa:", + "share modal title": "Partekatu bidalketa hau", + "share": "partekatu", + "unpin": "desfinkatu", +} diff --git a/l10n/strings.go b/l10n/strings.go index b31d6a8..43c2432 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -18,6 +18,8 @@ func Strings(lang string) map[string]string { return phrasesEO case "es": return phrasesES + case "eu": + return phrasesEU case "fa": return phrasesFA case "fr": From 8e58a42997fd48cae6afb418bb71bc32dc2cf889 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:52:59 -0400 Subject: [PATCH 50/76] Add Galician translations --- l10n/phrases_gl.go | 28 ++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 l10n/phrases_gl.go diff --git a/l10n/phrases_gl.go b/l10n/phrases_gl.go new file mode 100644 index 0000000..790944f --- /dev/null +++ b/l10n/phrases_gl.go @@ -0,0 +1,28 @@ +package l10n + +var phrasesGL = map[string]string{ + "Anonymous post": "Publicación anónima", + "Blogs": "Blogs", + "Enter": "Entrar", + "Newer": "Máis recente", + "Older": "Máis antigo", + "Posts": "Publicacións", + "Publish to...": "Publicar en...", + "Publish": "Publicar", + "Read more...": "Saber máis...", + "Subscribe": "Subscribir", + "This blog requires a password.": "Este blog require un contrasinal.", + "Toggle theme": "Cambiar decorado", + "View posts": "Ver Publicacións", + "delete": "eliminar", + "edit": "editar", + "email subscription prompt": "Escribe o teu email para recibir actualizacións.", + "move to...": "mover a...", + "pin": "fixar", + "published with write.as": "publicado con write.as", + "share modal ending": "Envíallo a un amigo, compárteo en internet e redes sociais. Saber máis.", + "share modal introduction": "Cada publicación ten un identificador, un URL único que podes compartir con calquera. Este é o URL:", + "share modal title": "Comparte a publicación", + "share": "compartir", + "unpin": "desafixar", +} diff --git a/l10n/strings.go b/l10n/strings.go index 43c2432..7b0bbdc 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -24,6 +24,8 @@ func Strings(lang string) map[string]string { return phrasesFA case "fr": return phrasesFR + case "gl": + return phrasesGL case "hu": return phrasesHU case "it": From 6594abfd76106472120f764520cb68ff8a6fc850 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:55:04 -0400 Subject: [PATCH 51/76] Update German translations --- l10n/phrases_de.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/l10n/phrases_de.go b/l10n/phrases_de.go index c06e8de..b0e9066 100644 --- a/l10n/phrases_de.go +++ b/l10n/phrases_de.go @@ -7,21 +7,23 @@ var phrasesDE = map[string]string{ "Newer": "Neuer", "Older": "Älter", "Posts": "Beiträge", - "Publish to...": "Veröffentlichen zu...", + "Publish to...": "Veröffentlichen zu", "Publish": "Veröffentlichen", "Read more...": "Weiterlesen...", + "Subscribe": "Abonnieren", "This blog requires a password.": "Dieser Blog benötigt ein Passwort", "Toggle theme": "Design ändern", "View posts": "Beiträge ansehen", "delete": "Löschen", "edit": "Bearbeiten", + "email subscription prompt": "Geben Sie Ihre E-Mail-Adresse ein, um Updates zu abonnieren.", "move to...": "Verschieben nach...", "pin": "Anheften", - "published with write.as": "veröffentlicht mit write.as", - "share modal ending": "Schicke es Freunden, teile es im Netz oder vielleicht tweete es. Lerne mehr.", - "share modal introduction": "Jeder veröffentlichte Beitrag hat eine geheime, einmalige URL, die du mit jedem teilen kannst. Die URL ist:", - "share modal title": "Teile diesen Beitrag", - "share": "Teilen", - "unpin": "Lösen", - "title dash": "–", + "published with write.as": "Veröffentlicht mit write.as", + "share modal ending": "Schicke es Freunden, teile es im Netz oder vielleicht tweete es. Lerne mehr.", + "share modal introduction": "Jeder veröffentlichte Beitrag hat eine geheime, einmalige URL, die du mit jedem teilen kannst. Die URL ist:", + "share modal title": "Teile diesen Beitrag", + "share": "Teilen", + "unpin": "Lösen", + "title dash": "–", } From 4ea5f505e135da6f345c7e8606902376d1f0408f Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:56:23 -0400 Subject: [PATCH 52/76] Update Greek translations --- l10n/phrases_el.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/l10n/phrases_el.go b/l10n/phrases_el.go index 8b8d96b..4bd4982 100644 --- a/l10n/phrases_el.go +++ b/l10n/phrases_el.go @@ -10,17 +10,19 @@ var phrasesEL = map[string]string{ "Publish to...": "Δημοσίευση στο...", "Publish": "Δημοσίευση", "Read more...": "Διαβάστε περισσότερα...", + "Subscribe": "Εγγραφείτε", "This blog requires a password.": "Αυτό το ιστολόγιο απαιτεί κωδικό.", "Toggle theme": "Αλλαγή θέματος", "View posts": "Προβολή Δημοσιεύσεων", "delete": "διαγραφή", "edit": "επεξεργασία", + "email subscription prompt": "Εισάγετε το email σας για να εγγραφείτε στις ενημερώσεις.", "move to...": "μετακίνηση στο...", "pin": "καρφίτσωμα", - "published with write.as": "δημοσιεύθηκε με το write.as", - "share modal ending": "Στείλτε το σε έναν φίλο, μοιραστείτε το στο διαδίκτυο ή κάντε το tweet. Μάθετε περισσότερα.", - "share modal introduction": "Κάθε αναρτημένη δημοσίευση έχει ένα κρυφό, μοναδικό σύνδεσμο που μπορείτε να μοιραστείτε με οποιονδήποτε. Αυτός είναι ο εν λόγω σύνδεσμος:", - "share modal title": "Μοιραστείτε αυτή τη δημοσίευση", - "share": "διαμοιρασμός", - "unpin": "ξεκαρφίτσωμα", + "published with write.as": "δημοσιεύθηκε με το write.as", + "share modal ending": "Στείλτε το σε έναν φίλο, μοιραστείτε το στο διαδίκτυο ή κάντε το tweet. Μάθετε περισσότερα.", + "share modal introduction": "Κάθε αναρτημένη δημοσίευση έχει ένα κρυφό, μοναδικό σύνδεσμο που μπορείτε να μοιραστείτε με οποιονδήποτε. Αυτός είναι ο εν λόγω σύνδεσμος:", + "share modal title": "Μοιραστείτε αυτή τη δημοσίευση", + "share": "διαμοιρασμός", + "unpin": "ξεκαρφίτσωμα", } From b0d08b0b2de64e38f8a1c780caba71df7bdcc816 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:57:53 -0400 Subject: [PATCH 53/76] Add Hebrew translations --- l10n/phrases_he.go | 28 ++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 l10n/phrases_he.go diff --git a/l10n/phrases_he.go b/l10n/phrases_he.go new file mode 100644 index 0000000..8d43ac3 --- /dev/null +++ b/l10n/phrases_he.go @@ -0,0 +1,28 @@ +package l10n + +var phrasesHE = map[string]string{ + "Anonymous post": "פוסטים אנונימיים", + "Blogs": "בלוגים", + "Enter": "היכנס", + "Newer": "חדש יותר", + "Older": "ישן יותר", + "Posts": "פוסטים", + "Publish to...": "פירסום ל...", + "Publish": "פירסום", + "Read more...": "קרא עוד...", + "Subscribe": "הירשם", + "This blog requires a password.": "לבלוג זה ישנה סיסמה", + "Toggle theme": "החלפת ערכת נושא", + "View posts": "צפייה בפוסטים", + "delete": "מחיקה", + "edit": "עריכה", + "email subscription prompt": "הכנס את המייל שלך כדי לקבל לעידכונים", + "move to...": "העברה ל...", + "pin": "הצמדה", + "published with write.as": "פורסם ע''י write.as", + "share modal ending": "שלח לחברים, פרסם ברשת, או אולי צייץ את זה. למד עוד.", + "share modal introduction": "לכל פוסט יש קישור סודי ויחודי שניתן לשתף עם כל אחד. הנה הקישור:", + "share modal title": "שיתוף פוסט זה", + "share": "שיתוף", + "unpin": "ביטול הצמדה", +} diff --git a/l10n/strings.go b/l10n/strings.go index 7b0bbdc..5481f29 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -26,6 +26,8 @@ func Strings(lang string) map[string]string { return phrasesFR case "gl": return phrasesGL + case "he": + return phrasesHE case "hu": return phrasesHU case "it": From 482d2a034048aba67c8f398ceee8118722c184ec Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 10:59:12 -0400 Subject: [PATCH 54/76] Update Italian translations --- l10n/phrases_it.go | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/l10n/phrases_it.go b/l10n/phrases_it.go index 378e224..63fa30b 100644 --- a/l10n/phrases_it.go +++ b/l10n/phrases_it.go @@ -1,24 +1,28 @@ package l10n var phrasesIT = map[string]string{ - "Anonymous post": "Post anonimo", - "Blogs": "Blogs", - "Newer": "Più recenti", - "Older": "Più vecchi", - "Posts": "Posts", - "Publish to...": "Pubblica su...", - "Publish": "Pubblica", - "Read more...": "Continua...", - "Toggle theme": "Attiva tema", - "View posts": "Vedi Posts", - "delete": "cancella", - "edit": "modifica", - "move to...": "sposta verso...", - "pin": "attacca", - "published with write.as": "pubblicato con write.as", - "share modal ending": "Mandalo ad un amico, condividilo sul web, oppure tweettalo. Per saperne di più.", - "share modal introduction": "Ogni post pubblicato possiede un URL segreto e unico che puoi condividere con chiunque. Questo è l'URL:", - "share modal title": "Condividi questo post", - "share": "condividi", - "unpin": "stacca", + "Anonymous post": "Post anonimo", + "Blogs": "Blogs", + "Enter": "Invio", + "Newer": "Recenti", + "Older": "Precedenti", + "Posts": "Posts", + "Publish to...": "Pubblica su...", + "Publish": "Pubblica", + "Read more...": "Continua...", + "Subscribe": "Sottoscrivi", + "This blog requires a password.": "Questo blog necessita una password.", + "Toggle theme": "Attiva tema", + "View posts": "Vedi Posts", + "delete": "cancella", + "edit": "modifica", + "email subscription prompt": "Inserisci la tua email per ricevere aggiornamenti", + "move to...": "sposta verso...", + "pin": "appunta", + "published with write.as": "pubblicato con write.as", + "share modal ending": "Mandala ad un amico, condividila sul web, oppure tweettala. Per saperne di più.", + "share modal introduction": "Ogni post pubblicato possiede una URL segreta e unica che puoi condividere con chiunque. Questa è l'URL:", + "share modal title": "Condividi questo post", + "share": "condividi", + "unpin": "stacca", } From 789795d7fba6669813d0479e53bb81b62965523e Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 5 Nov 2021 11:00:54 -0400 Subject: [PATCH 55/76] Update Japanese translations --- l10n/phrases_ja.go | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/l10n/phrases_ja.go b/l10n/phrases_ja.go index 0f3868b..7050ea1 100644 --- a/l10n/phrases_ja.go +++ b/l10n/phrases_ja.go @@ -1,24 +1,28 @@ package l10n var phrasesJA = map[string]string{ - "Anonymous post": "匿名投稿", - "Blogs": "ブログ", - "Newer": "新しい投稿", - "Older": "古い投稿", - "Posts": "投稿", - "Publish to...": "公開先…", - "Publish": "公開", - "Read more...": "もっと読む…", - "Toggle theme": "テーマを変更", - "View posts": "投稿を見る", - "delete": "削除", - "edit": "編集", - "move to...": "移動…", - "pin": "固定表示する", - "published with write.as": "write.as を使って公開されました", - "share modal ending": "友達に送信したり、ウェブの大海にシェアしたり、ツイートしたり。もっと知る。", - "share modal introduction": "全ての投稿には秘密の、シェアするとだれでも見ることのできる固有のURLがあります。これがそのURLです:", - "share modal title": "投稿をシェアする", - "share": "シェア", - "unpin": "固定表示をやめる", + "Anonymous post": "匿名投稿", + "Blogs": "ブログ", + "Enter": "認証", + "Newer": "新しい投稿", + "Older": "古い投稿", + "Posts": "投稿", + "Publish to...": "公開先…", + "Publish": "公開", + "Read more...": "もっと読む…", + "Subscribe": "購読", + "This blog requires a password.": "このブログはパスワードを必要としています。", + "Toggle theme": "テーマを変更", + "View posts": "投稿を見る", + "delete": "削除", + "edit": "編集", + "email subscription prompt": "このブログを購読したい場合は、メールアドレスを入力してください。", + "move to...": "移動…", + "pin": "固定表示する", + "published with write.as": "write.as を使って公開されました", + "share modal ending": "友達に送信したり、Web 上で共有したり、ツイートすることが出来ます。もっと詳しく。", + "share modal introduction": "全ての投稿には秘密の、シェアするとだれでも見ることのできる固有の URL があります。これがその URL です:", + "share modal title": "投稿をシェアする", + "share": "シェア", + "unpin": "固定表示をやめる", } From 251490523da5d4774e10c533ba5f27a079f388d1 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 15 Dec 2021 15:57:19 -0500 Subject: [PATCH 56/76] Add ApplyBasicAccessibleMarkdown func --- posts/render.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/posts/render.go b/posts/render.go index d0885d5..5fede8f 100644 --- a/posts/render.go +++ b/posts/render.go @@ -63,6 +63,31 @@ func ApplyBasicMarkdown(data []byte) string { return outHTML } +// ApplyBasicAccessibleMarkdown applies Markdown to the given data, rendering basic text formatting and preserving hard +// line breaks. It is meant for formatting text in small, multi-line UI elements, like user profile biographies. +func ApplyBasicAccessibleMarkdown(data []byte) string { + mdExtensions := 0 | + blackfriday.EXTENSION_STRIKETHROUGH | + blackfriday.EXTENSION_SPACE_HEADERS | + blackfriday.EXTENSION_HEADER_IDS | + blackfriday.EXTENSION_HARD_LINE_BREAK + htmlFlags := 0 | + blackfriday.HTML_SKIP_HTML | + blackfriday.HTML_USE_SMARTYPANTS | + blackfriday.HTML_SMARTYPANTS_DASHES + + // Generate Markdown + md := blackfriday.Markdown([]byte(data), blackfriday.HtmlRenderer(htmlFlags, "", ""), mdExtensions) + // Strip out bad HTML + policy := bluemonday.UGCPolicy() + policy.AllowAttrs("class", "id").Globally() + outHTML := string(policy.SanitizeBytes(md)) + outHTML = markeddownReg.ReplaceAllString(outHTML, "$1") + outHTML = strings.TrimRightFunc(outHTML, unicode.IsSpace) + + return outHTML +} + // StripHTMLWithoutEscaping strips HTML tags with bluemonday's StrictPolicy, then unescapes the HTML // entities added in by sanitizing the content. func StripHTMLWithoutEscaping(content string) string { From 3e1dcd609c45213d3a46fc3d0969f475c95266f8 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 6 Jan 2022 13:35:11 -0500 Subject: [PATCH 57/76] Restore default title dash --- l10n/phrases.go | 1 + 1 file changed, 1 insertion(+) diff --git a/l10n/phrases.go b/l10n/phrases.go index b1c10cb..21c3e23 100644 --- a/l10n/phrases.go +++ b/l10n/phrases.go @@ -38,4 +38,5 @@ var phrases = map[string]string{ "share modal title": "Share this post", "share": "share", "unpin": "unpin", + "title dash": "—", } From 0da0bcaf018e72a51453317e1bd06b8b29808d21 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 18 Jan 2022 16:27:28 -0500 Subject: [PATCH 58/76] Support HTML links with rel attr in ApplyBasicAccessibleMarkdown Ref T874 T744 --- posts/render.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/posts/render.go b/posts/render.go index 5fede8f..ce9d5b0 100644 --- a/posts/render.go +++ b/posts/render.go @@ -64,7 +64,7 @@ func ApplyBasicMarkdown(data []byte) string { } // ApplyBasicAccessibleMarkdown applies Markdown to the given data, rendering basic text formatting and preserving hard -// line breaks. It is meant for formatting text in small, multi-line UI elements, like user profile biographies. +// line breaks in HTML. It is meant for formatting text in small, multi-line UI elements, like user profile biographies. func ApplyBasicAccessibleMarkdown(data []byte) string { mdExtensions := 0 | blackfriday.EXTENSION_STRIKETHROUGH | @@ -72,8 +72,8 @@ func ApplyBasicAccessibleMarkdown(data []byte) string { blackfriday.EXTENSION_HEADER_IDS | blackfriday.EXTENSION_HARD_LINE_BREAK htmlFlags := 0 | - blackfriday.HTML_SKIP_HTML | blackfriday.HTML_USE_SMARTYPANTS | + blackfriday.HTML_USE_XHTML | blackfriday.HTML_SMARTYPANTS_DASHES // Generate Markdown @@ -81,7 +81,10 @@ func ApplyBasicAccessibleMarkdown(data []byte) string { // Strip out bad HTML policy := bluemonday.UGCPolicy() policy.AllowAttrs("class", "id").Globally() + policy.AllowAttrs("rel").OnElements("a") + policy.RequireNoFollowOnLinks(false) outHTML := string(policy.SanitizeBytes(md)) + // Strip surrounding

tags that blackfriday adds outHTML = markeddownReg.ReplaceAllString(outHTML, "$1") outHTML = strings.TrimRightFunc(outHTML, unicode.IsSpace) From ab8ed5dd58b20cc5bf85e273cbe8a3962f3b54bf Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 11 May 2022 18:06:23 -0400 Subject: [PATCH 59/76] Support sending out activities around Persons --- activitystreams/activity.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index 93a40af..ebb118f 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -106,19 +106,29 @@ func NewFollowActivity(actorIRI, followeeIRI string) *FollowActivity { // Object is the primary base type for the Activity Streams vocabulary. type Object struct { BaseObject - Published time.Time `json:"published"` + Published time.Time `json:"published,omitempty"` Summary *string `json:"summary,omitempty"` - InReplyTo *string `json:"inReplyTo"` + InReplyTo *string `json:"inReplyTo,omitempty"` URL string `json:"url"` - AttributedTo string `json:"attributedTo"` - To []string `json:"to"` + AttributedTo string `json:"attributedTo,omitempty"` + To []string `json:"to,omitempty"` CC []string `json:"cc,omitempty"` Name string `json:"name,omitempty"` - Content string `json:"content"` + Content string `json:"content,omitempty"` ContentMap map[string]string `json:"contentMap,omitempty"` - Tag []Tag `json:"tag"` + Tag []Tag `json:"tag,omitempty"` Attachment []Attachment `json:"attachment,omitempty"` + // Person + Inbox string `json:"inbox,omitempty"` + Outbox string `json:"outbox,omitempty"` + Following string `json:"following,omitempty"` + Followers string `json:"followers,omitempty"` + PreferredUsername string `json:"preferredUsername,omitempty"` + Icon *Image `json:"icon,omitempty"` + PublicKey *PublicKey `json:"publicKey,omitempty"` + Endpoints *Endpoints `json:"endpoints,omitempty"` + // Extensions // NOTE: add extensions here } @@ -150,3 +160,13 @@ func NewArticleObject() *Object { } return &o } + +// NewPersonObject creates a basic Person object. +func NewPersonObject() *Object { + o := Object{ + BaseObject: BaseObject{ + Type: "Person", + }, + } + return &o +} From 73b029dffaf3e6396da06fd104adc5f53a09e4f7 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Fri, 11 Nov 2022 01:56:53 -0500 Subject: [PATCH 60/76] Support decoding PKCS#8 keys These are generated by OpenSSL v3. Previously, key decoding would fail, breaking federation. This fixes #14 --- activitypub/keys.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitypub/keys.go b/activitypub/keys.go index 9540668..51ae2ec 100644 --- a/activitypub/keys.go +++ b/activitypub/keys.go @@ -69,8 +69,8 @@ func parsePublicKey(der []byte) (crypto.PublicKey, error) { // them in that order. func DecodePrivateKey(k []byte) (crypto.PrivateKey, error) { block, _ := pem.Decode(k) - if block == nil || block.Type != "RSA PRIVATE KEY" { - return nil, fmt.Errorf("failed to decode PEM block containing private key") + if block == nil || (block.Type != "RSA PRIVATE KEY" && block.Type != "PRIVATE KEY") { + return nil, fmt.Errorf("failed to decode PEM block containing private key, type %s", block.Type) } return parsePrivateKey(block.Bytes) From 9e2b7620b101f9b0d7352ce313eb90714f5118c5 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 4 Apr 2023 12:09:49 -0400 Subject: [PATCH 61/76] Add funcs PostTitle, GetSlug, GetSlugFromPost --- go.mod | 2 +- posts/parse.go | 46 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5170100..1955354 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.5 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect - github.com/writeas/go-strip-markdown v2.0.1+incompatible + github.com/writeas/go-strip-markdown/v2 v2.1.1 github.com/writeas/impart v1.1.1 github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.7.1 diff --git a/posts/parse.go b/posts/parse.go index 168289f..0d61bac 100644 --- a/posts/parse.go +++ b/posts/parse.go @@ -2,7 +2,8 @@ package posts import ( "fmt" - stripmd "github.com/writeas/go-strip-markdown" + stripmd "github.com/writeas/go-strip-markdown/v2" + "github.com/writeas/slug" "github.com/writeas/web-core/stringmanip" "regexp" "strings" @@ -37,6 +38,20 @@ func ExtractTitle(content string) (title string, body string) { return } +func PostTitle(content, friendlyId string) string { + content = StripHTMLWithoutEscaping(content) + + content = strings.TrimLeftFunc(stripmd.Strip(content), unicode.IsSpace) + eol := strings.IndexRune(content, '\n') + blankLine := strings.Index(content, "\n\n") + if blankLine != -1 && blankLine <= eol && blankLine <= assumedTitleLen { + return strings.TrimSpace(content[:blankLine]) + } else if utf8.RuneCountInString(content) <= maxTitleLen { + return content + } + return friendlyId +} + func FriendlyPostTitle(content, friendlyId string) string { content = StripHTMLWithoutEscaping(content) @@ -163,3 +178,32 @@ func PostLede(t string, includePunc bool) string { return t } + +func GetSlug(title, lang string) string { + return GetSlugFromPost("", title, lang) +} + +func GetSlugFromPost(title, body, lang string) string { + if title == "" { + // Remove Markdown, so e.g. link URLs and image alt text don't make it into the slug + body = strings.TrimSpace(stripmd.StripOptions(body, stripmd.Options{SkipImages: true})) + title = PostTitle(body, body) + } + title = PostLede(title, false) + // Truncate lede if needed + title, _ = TruncToWord(title, maxTitleLen) + var s string + if lang != "" && len(lang) == 2 { + s = slug.MakeLang(title, lang) + } else { + s = slug.Make(title) + } + + // Transliteration may cause the slug to expand past the limit, so truncate again + s, _ = TruncToWord(s, maxTitleLen) + return strings.TrimFunc(s, func(r rune) bool { + // TruncToWord doesn't respect words in a slug, since spaces are replaced + // with hyphens. So remove any trailing hyphens. + return r == '-' + }) +} From 4e8d67e937c97ad6651dc404c660ff092c4b5c37 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 4 Apr 2023 12:10:20 -0400 Subject: [PATCH 62/76] Update go.sum with go-strip-markdown --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index e02806f..3439cdc 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= +github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28= +github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA= github.com/writeas/impart v1.1.1 h1:RyA9+CqbdbDuz53k+nXCWUY+NlEkdyw6+nWanxSBl5o= github.com/writeas/impart v1.1.1/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y= github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjFc5o= From 4934315624487b3cf3cf997b399d3062e83b2af8 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Tue, 4 Apr 2023 12:10:40 -0400 Subject: [PATCH 63/76] Reformat example code on PostDescription --- posts/parse.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/posts/parse.go b/posts/parse.go index 0d61bac..6c179ce 100644 --- a/posts/parse.go +++ b/posts/parse.go @@ -76,12 +76,13 @@ func FriendlyPostTitle(content, friendlyId string) string { // choosing what to generate. In case a post has a title, this function will // fail, and logic should instead be implemented to skip this when there's no // title, like so: -// var desc string -// if title == "" { -// desc = PostDescription(content, title, friendlyId) -// } else { -// desc = ShortPostDescription(content) -// } +// +// var desc string +// if title == "" { +// desc = PostDescription(content, title, friendlyId) +// } else { +// desc = ShortPostDescription(content) +// } func PostDescription(content, title, friendlyId string) string { maxLen := 140 From 33b431fb4187d3bc7eb17388a308ba943f6eef9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Apr 2023 16:11:47 +0000 Subject: [PATCH 64/76] Bump github.com/microcosm-cc/bluemonday from 1.0.5 to 1.0.23 Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.5 to 1.0.23. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.5...v1.0.23) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 47 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 1955354..9e9f99f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.10 require ( github.com/gofrs/uuid v3.3.0+incompatible github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec - github.com/microcosm-cc/bluemonday v1.0.5 + github.com/microcosm-cc/bluemonday v1.0.23 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/writeas/go-strip-markdown/v2 v2.1.1 @@ -13,7 +13,7 @@ require ( github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.7.1 github.com/writeas/slug v1.2.0 - golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect ) diff --git a/go.sum b/go.sum index 3439cdc..d5552a3 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU= -github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -13,14 +11,12 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec h1:ZXWuspqypleMuJy4bzYEqlMhJnGAYpLrWe5p7W3CdvI= github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec/go.mod h1:voECJzdraJmolzPBgL9Z7ANwXf4oMXaTCsIkdiPpR/g= -github.com/microcosm-cc/bluemonday v1.0.5 h1:cF59UCKMmmUgqN1baLvqU/B1ZsMori+duLVTLpgiG3w= -github.com/microcosm-cc/bluemonday v1.0.5/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w= +github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= +github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ= github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw= -github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE= github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28= github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA= github.com/writeas/impart v1.1.1 h1:RyA9+CqbdbDuz53k+nXCWUY+NlEkdyw6+nWanxSBl5o= @@ -31,15 +27,42 @@ github.com/writeas/saturday v1.7.1 h1:lYo1EH6CYyrFObQoA9RNWHVlpZA5iYL5Opxo7PYAnZ github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ= github.com/writeas/slug v1.2.0 h1:EMQ+cwLiOcA6EtFwUgyw3Ge18x9uflUnOnR6bp/J+/g= github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQFubfQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= -golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= From edb6b3fe63eafc2aab29fd7caacab43467c44870 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 6 Jul 2023 17:50:20 -0400 Subject: [PATCH 65/76] Re-gofmt bots.go --- bots/bots.go | 194 +++++++++++++++++++++++++-------------------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/bots/bots.go b/bots/bots.go index 651a22d..f3f4f36 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -6,113 +6,113 @@ package bots import "strings" var bots = map[string]bool{ - "ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true, - "AddThis.com robot tech.support@clearspring.com": true, - "Alwyzbot/1.0": true, - "Apache-HttpClient/4.2.2 (java 1.5)": true, - "bitlybot": true, - "bitlybot/3.0 (+http://bit.ly/)": true, + "ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true, + "AddThis.com robot tech.support@clearspring.com": true, + "Alwyzbot/1.0": true, + "Apache-HttpClient/4.2.2 (java 1.5)": true, + "bitlybot": true, + "bitlybot/3.0 (+http://bit.ly/)": true, "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)": true, - "cis455crawler": true, - "COMODO SSL Checker": true, - "crawlernutchtest/Nutch-1.9": true, + "cis455crawler": true, + "COMODO SSL Checker": true, + "crawlernutchtest/Nutch-1.9": true, "CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true, "CSS Certificate Spider (http://www.css-security.com/certificatespider/)": true, - "datebot": true, - "DeadYetBot/1.0 (+http://deadyet.lol)": true, + "datebot": true, + "DeadYetBot/1.0 (+http://deadyet.lol)": true, "dj-research/Nutch-1.11 (analytics@@demandjump.com)": true, "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true, "DomainCrawler/1.0": true, "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com": true, - "ExactSeekCrawler/1.0": true, - "Gigabot/1.0": true, - "Go 1.1 package http": true, - "Googlebot/2.1 (+http://www.googlebot.com/bot.html)": true, - "Googlebot/2.1 (+http://www.google.com/bot.html)": true, - "Googlebot-Image/1.0": true, - "libwww-perl/6.04": true, + "ExactSeekCrawler/1.0": true, + "Gigabot/1.0": true, + "Go 1.1 package http": true, + "Googlebot/2.1 (+http://www.googlebot.com/bot.html)": true, + "Googlebot/2.1 (+http://www.google.com/bot.html)": true, + "Googlebot-Image/1.0": true, + "libwww-perl/6.04": true, "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)": true, "LivelapBot/0.2 (http://site.livelap.com/crawler)": true, "LSSRocketCrawler/1.0 LightspeedSystems": true, "Mediatoolkitbot (complaints@mediatoolkit.com)": true, "Mediatoolkitbot (info@mediatoolkit.com)": true, "Mediumbot-MetaTagFetcher/0.1 (+https://medium.com/)": true, - "Melvil/1.0": true, - "MetaCommentBot; http://metacomment.io/about": true, - "mfibot/1.1 (http://www.mfisoft.ru/analyst/; ; en-RU)": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true, - "Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true, - "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true, - "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true, - "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, - "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, - "Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true, - "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true, - "Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true, - "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true, - "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true, - "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true, - "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true, - "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true, - "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true, - "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true, - "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true, - "Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true, - "Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true, - "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true, - "Mozilla/5.0 (compatible; LinkFeatureBot)": true, - "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true, - "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true, - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true, - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true, - "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true, - "Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true, - "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true, - "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true, - "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true, - "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true, - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true, - "Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true, - "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true, - "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true, - "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true, - "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true, - "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true, - "Mozilla/5.0 (compatible; OpsBot/1.0)": true, - "Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true, - "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true, - "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true, - "Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true, - "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true, - "Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true, - "Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true, - "Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true, - "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true, - "Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true, - "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true, - "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true, - "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true, - "Mozilla/5.0 (compatible; TestCrawler)": true, - "Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true, - "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, - "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, - "Mozilla/5.0 (compatible; woriobot +http://worio.com)": true, - "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true, - "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true, - "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true, - "Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true, - "Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true, + "Melvil/1.0": true, + "MetaCommentBot; http://metacomment.io/about": true, + "mfibot/1.1 (http://www.mfisoft.ru/analyst/; ; en-RU)": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true, + "Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true, + "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true, + "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true, + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, + "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, + "Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true, + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true, + "Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true, + "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true, + "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true, + "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true, + "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true, + "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true, + "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true, + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true, + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true, + "Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true, + "Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true, + "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true, + "Mozilla/5.0 (compatible; LinkFeatureBot)": true, + "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true, + "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true, + "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true, + "Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true, + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true, + "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true, + "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true, + "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true, + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true, + "Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true, + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true, + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true, + "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true, + "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true, + "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true, + "Mozilla/5.0 (compatible; OpsBot/1.0)": true, + "Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true, + "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true, + "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true, + "Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true, + "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true, + "Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true, + "Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true, + "Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true, + "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true, + "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true, + "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true, + "Mozilla/5.0 (compatible; TestCrawler)": true, + "Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true, + "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, + "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, + "Mozilla/5.0 (compatible; woriobot +http://worio.com)": true, + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true, + "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true, + "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true, + "Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true, + "Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true, @@ -176,13 +176,13 @@ var bots = map[string]bool{ "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3": true, "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1": true, "Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13": true, - "myspider/Nutch-1.10": true, + "myspider/Nutch-1.10": true, "netEstate NE Crawler (+http://www.website-datenbank.de/)": true, "omgili/0.5 +http://omgili.com": true, "PercolateCrawler/4 (ops@percolate.com)": true, "python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-43-generic": true, "python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-61-generic": true, - "Python-urllib/2.7": true, + "Python-urllib/2.7": true, "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true, "Ruby": true, "SafeDNSBot (https://www.safedns.com/searchbot)": true, @@ -202,7 +202,7 @@ var bots = map[string]bool{ "Stratagems Kumo": true, "Traackr.com Bot": true, "voltron": true, - "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, + "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true, "yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true, "YisouSpider": true, From dee1e7cfbb41879ebd155deef1ad4286e9e74ffe Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 6 Jul 2023 18:29:39 -0400 Subject: [PATCH 66/76] Update bots list Also add and sort prefixes --- bots/bots.go | 515 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 371 insertions(+), 144 deletions(-) diff --git a/bots/bots.go b/bots/bots.go index f3f4f36..1b8d58c 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -6,141 +6,285 @@ package bots import "strings" var bots = map[string]bool{ - "ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true, - "AddThis.com robot tech.support@clearspring.com": true, - "Alwyzbot/1.0": true, - "Apache-HttpClient/4.2.2 (java 1.5)": true, - "bitlybot": true, - "bitlybot/3.0 (+http://bit.ly/)": true, + "ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true, + "AcademicBotRTU (https://academicbot.rtu.lv; mailto:caps@rtu.lv)": true, + "AddThis.com robot tech.support@clearspring.com": true, + "AdsBot-Google (+http://www.google.com/adsbot.html)": true, + "AdsTxtCrawlerTP/1.2": true, + "Alwyzbot/1.0": true, + "Amazon-Advertising-ad-standards-bot/1.0": true, + "Apple Color Emoji": true, + "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)": true, + "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)": true, + "BeeperBot/0 Matrix-Media-Repo/1": true, + "bidswitchbot/1.0": true, + "bitlybot": true, + "BrightEdge Crawler/1.0 (crawler@brightedge.com)": true, + "BSbot 1.1 (monthly copyright check - html/js/css)": true, "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)": true, - "cis455crawler": true, - "COMODO SSL Checker": true, - "crawlernutchtest/Nutch-1.9": true, - "CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true, + "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)": true, + "Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "cis455crawler": true, + "Clickagy Intelligence Bot v2": true, + "COMODO SSL Checker": true, + "companyspotter/1.0.0.0 (robot@companyspotter.com)": true, + "crawler_eb_germany_2.0": true, + "crawlernutchtest/Nutch-1.9": true, + "CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true, "CSS Certificate Spider (http://www.css-security.com/certificatespider/)": true, - "datebot": true, - "DeadYetBot/1.0 (+http://deadyet.lol)": true, - "dj-research/Nutch-1.11 (analytics@@demandjump.com)": true, + "datebot": true, + "DeadYetBot/1.0 (+http://deadyet.lol)": true, + "Deskyobot/1.0 (+https://www.deskyo.com/bot)": true, + "DingTalkBot-LinkService/1.0 (+https://open-doc.dingtalk.com/microapp/faquestions/ftpfeu)": true, + "DisqusAdstxtCrawler/0.1 (+https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide)": true, + "dj-research/Nutch-1.11 (analytics@@demandjump.com)": true, + "DnBCrawler-Analytics": true, "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true, + "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com": true, "DomainCrawler/1.0": true, - "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com": true, + "DomainStatsBot/1.0 (https://domainstats.com/pages/our-bot)": true, + "drinespider/Nutch-1.19 (D-RINE Spider; www.d-rine.com/search/about; www.d-rine.com/contact)": true, + "DuckDuckBot-Https/1.1; (+https://duckduckgo.com/duckduckbot)": true, "ExactSeekCrawler/1.0": true, - "Gigabot/1.0": true, - "Go 1.1 package http": true, - "Googlebot/2.1 (+http://www.googlebot.com/bot.html)": true, - "Googlebot/2.1 (+http://www.google.com/bot.html)": true, - "Googlebot-Image/1.0": true, - "libwww-perl/6.04": true, - "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)": true, - "LivelapBot/0.2 (http://site.livelap.com/crawler)": true, - "LSSRocketCrawler/1.0 LightspeedSystems": true, - "Mediatoolkitbot (complaints@mediatoolkit.com)": true, - "Mediatoolkitbot (info@mediatoolkit.com)": true, - "Mediumbot-MetaTagFetcher/0.1 (+https://medium.com/)": true, + "EyeMonIT Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36": true, + "FediCrawl/1.0": true, + "fedistatsCrawler/1.0": true, + "FeedlyBot/1.0 (http://feedly.com)": true, + "GG PeekBot 2.0 ( https://www.gg.pl/ https://www.gg.pl/info/praca/ )": true, + "Gigabot/1.0": true, + "GNUsocialBot 2.0.1-beta0 - https://gnusocial.rocks": true, + "Go 1.1 package http": true, + "Go-http-client/2.0": true, + "Google-Adwords-Instant (+http://www.google.com/adsbot.html)": true, + "Google-Display-Ads-Bot": true, + "Googlebot": true, + "Greppr Bot 1.0/Nutch-1.19": true, + "GroupMeBot/1.0": true, + "GuzzleHttp/7": true, + "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot": true, + "HubSpot Connect 2.0 (http://dev.hubspot.com/) (namespace: hs_web_crawler) - RoadsModelsJobs-hubspot-53-domain-web-crawl": true, + "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)": true, + "IABot/2.0 (+https://meta.wikimedia.org/wiki/InternetArchiveBot/FAQ_for_sysadmins) (Checking if link from Wikipedia is broken and needs removal)": true, + "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)": true, + "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)": true, + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)": true, + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)": true, + "LivelapBot/0.2 (http://site.livelap.com/crawler)": true, + "LSSRocketCrawler/1.0 LightspeedSystems": true, + "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)": true, + "Mastodon server indexer": true, + "MBCrawler/1.0 (https://monitorbacklinks.com/robot)": true, + "Mediatoolkitbot (complaints@mediatoolkit.com)": true, + "Mediatoolkitbot (info@mediatoolkit.com)": true, "Melvil/1.0": true, "MetaCommentBot; http://metacomment.io/about": true, "mfibot/1.1 (http://www.mfisoft.ru/analyst/; ; en-RU)": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true, - "Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true, - "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true, - "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true, - "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, - "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, - "Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true, - "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true, - "Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true, - "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true, - "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true, - "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true, - "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true, - "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true, - "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true, - "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true, - "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true, - "Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true, - "Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true, - "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true, - "Mozilla/5.0 (compatible; LinkFeatureBot)": true, - "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true, - "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true, - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true, - "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true, - "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true, - "Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true, - "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true, - "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true, - "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true, - "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true, - "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true, - "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true, - "Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true, - "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true, - "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true, - "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true, - "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true, - "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true, - "Mozilla/5.0 (compatible; OpsBot/1.0)": true, - "Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true, - "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true, - "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true, - "Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true, - "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true, - "Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true, - "Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true, - "Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true, - "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true, - "Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true, - "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true, - "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true, - "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true, - "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true, - "Mozilla/5.0 (compatible; TestCrawler)": true, - "Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true, - "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, - "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, - "Mozilla/5.0 (compatible; woriobot +http://worio.com)": true, - "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true, - "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true, - "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true, - "Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true, - "Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:42.0) Gecko/20100101 Firefox/42.0": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)": true, - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:29.0) Gecko/20100101 Firefox/29.0": true, - "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5": true, + "Minoru's Fediverse Crawler (+https://nodes.fediverse.party)": true, + "mj12bot": true, + "Mozilla/4.0 (compatible; QwertyNetworksBot/1.0; +https://qwertynetworks.com/faq/QwertyNetworksBot)": true, + "Mozilla/4.0 (compatible; Wibybot; https://wiby.me/)": true, + "Mozilla/5.0 (Android 11; Mobile; rv:109.0) Gecko/114.0 Firefox/114.0": true, + "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)": true, + "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)": true, + "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true, + "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true, + "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)": true, + "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot) Zeno/0030699 warc/v0.8.32": true, + "Mozilla/5.0 (compatible; Atomseobot/2.0; +http://https://error404.atomseo.com/)": true, + "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)": true, + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html": true, + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, + "Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, + "Mozilla/5.0 (compatible; BitSightBot/1.0)": true, + "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, + "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)": true, + "Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true, + "Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.0.0 Safari/537.36": true, + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true, + "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)": true, + "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)": true, + "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)": true, + "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)": true, + "Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true, + "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)": true, + "Mozilla/5.0 (compatible; Dmbot/1.1)": true, + "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true, + "Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)": true, + "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true, + "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)": true, + "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true, + "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true, + "Mozilla/5.0 (compatible; FFZBot/2.4.1; +https://www.frankerfacez.com)": true, + "Mozilla/5.0 (compatible; FFZBot/3.0.0; +https://www.frankerfacez.com)": true, + "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true, + "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true, + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)": true, + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true, + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true, + "Mozilla/5.0 (compatible; Hkfl-Bot/%s +https://hackerfall.com/)": true, + "Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)": true, + "Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true, + "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)": true, + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)": true, + "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true, + "Mozilla/5.0 (compatible; LinkFeatureBot)": true, + "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true, + "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Img/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true, + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Target/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true, + "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true, + "Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true, + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true, + "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true, + "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)": true, + "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true, + "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)": true, + "Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)": true, + "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true, + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true, + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true, + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true, + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true, + "Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true, + "Mozilla/5.0 (compatible; NetpeakCheckerBot/3.6; +https://netpeaksoftware.com/checker)": true, + "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true, + "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true, + "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true, + "Mozilla/5.0 (compatible; OpsBot/1.0)": true, + "Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true, + "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true, + "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/hc/en-us/articles/360006695637-PaperLiBot)": true, + "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)": true, + "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true, + "Mozilla/5.0 (compatible; proximic; +https://www.comscore.com/Web-Crawler)": true, + "Mozilla/5.0 (compatible; Qwantify-dev/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev10230/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev19924/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev19967/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev2270/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev22709/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev31653/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev4541/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev5115/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-dev846/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify-worker2994/1.0; +https://help.qwant.com/bot/)": true, + "Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true, + "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true, + "Mozilla/5.0 (compatible; rss2tg bot; +http://komar.in/en/rss2tg_crawler)": true, + "Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true, + "Mozilla/5.0 (compatible; SearchMySiteBot/1.0; +https://searchmysite.net)": true, + "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SemrushBot; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true, + "Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true, + "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true, + "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true, + "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)": true, + "Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true, + "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true, + "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)": true, + "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)": true, + "Mozilla/5.0 (compatible; SynapseMediaProxyBot/0.0.1; https://git.pixie.town/f0x/synapse-media-proxy)": true, + "Mozilla/5.0 (compatible; TestCrawler)": true, + "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)": true, + "Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true, + "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, + "Mozilla/5.0 (compatible; WebwikiBot/2.1; +https://www.webwiki.de)": true, + "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)": true, + "Mozilla/5.0 (compatible; woriobot +http://worio.com)": true, + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) sieve.k8s.crawler-production/1688317128-0": true, + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true, + "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)": true, + "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0": true, + "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0": true, + "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true, + "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)": true, + "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)": true, + "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0": true, + "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots)": true, + "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true, + "Mozilla/5.0 (compatible; YetiBOT/0.1b; +http://yetibot.ovh)": true, + "Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true, + "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e YisouSpider/5.0 Safari/602.1": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone14,3__weibo__13.6.3__iphone__os15.1)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone12,8__weibo__13.6.3__iphone__os15.6)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)": true, + "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1": true, + "Mozilla/5.0 (Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0": true, + "Mozilla/5.0 (Linux x86_64; rv:114.0) Gecko/20100101 Firefox/114.0": true, + "Mozilla/5.0 (Linux; Android 10; BAH3-W59 Build/HUAWEIBAH3-W59; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.105 Safari/537.36 Weibo (HUAWEI-BAH3-W59__weibo__12.5.2__android__android10)": true, + "Mozilla/5.0 (Linux; Android 10; VCE-AL00 Build/HUAWEIVCE-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.105 Mobile Safari/537.36 Weibo (HUAWEI-VCE-AL00__weibo__13.6.3__android__android10)": true, + "Mozilla/5.0 (Linux; Android 13; V2244A Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.74 Mobile Safari/537.36V2244A_13_WeiboIntlAndroid_6200": true, + "Mozilla/5.0 (Linux; Android 13; V2302A Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.74 Mobile Safari/537.36 V2302A_13_WeiboIntlAndroid_2612": true, + "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)": true, + "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)": true, + "Mozilla/5.0 (Linux; Android 5.1.1; A37f) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36": true, + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.120 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Linux; Android 6.0.1; vivo 1606 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36 VivoBrowser/5.7.0.6": true, + "Mozilla/5.0 (Linux; Android 6.0.1; vivo Y66 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/87.0.4280.141 Mobile Safari/537.36 VivoBrowser/10.9.14.0": true, + "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)": true, + "Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:42.0) Gecko/20100101 Firefox/42.0": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:29.0) Gecko/20100101 Firefox/29.0": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.7 DuckDuckGo/7 Safari/605.1.15": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.81 Safari/537.36 Edg/104.0.1293.54; Bot": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36 webtru_crawler": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36": true, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 ISSCyberRiskCrawler/1.1.3": true, + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5": true, + "Mozilla/5.0 (Morningscore Bot/1.0)": true, "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0": true, - "Mozilla/5.0 (Windows; Crawler; U; Windows NT 6.0; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7 (.NET CLR 3.5.30729)": true, + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)": true, + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0": true, "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36": true, "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0": true, "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko": true, "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.27+ (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27": true, "Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0": true, "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2": true, - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true, "Mozilla/5.0 (Windows NT 6.1) (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)": true, + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true, + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36": true, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36": true, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36": true, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36": true, @@ -151,10 +295,11 @@ var bots = map[string]bool{ "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0": true, "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36": true, "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0": true, + "Mozilla/5.0 (Windows; Crawler; U; Windows NT 6.0; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7 (.NET CLR 3.5.30729)": true, "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.6) Gecko/20040206 Firefox/0.8": true, - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)": true, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler": true, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b4) Gecko/20050908 Firefox/1.4": true, + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)": true, "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja; rv:1.8.0.10) Gecko/20070216 Firefox/1.5.0.10": true, "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11": true, "Mozilla/5.0 (Windows; U; WinNT4.0; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1": true, @@ -163,8 +308,9 @@ var bots = map[string]bool{ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36/Gringe": true, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36": true, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36": true, - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20061201 Firefox/2.0.0.8": true, + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)": true, "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20060118 Firefox/1.5": true, + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20061201 Firefox/2.0.0.8": true, "Mozilla/5.0 (X11; U; Linux i686; fr-FR; rv:1.8.1.6) Gecko/20080208 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12": true, "Mozilla/5.0 (X11; U; Linux i686; fr; rv:1.9.0.9) Gecko/2009042113 Ubuntu/8.04 (hardy) Firefox/3.0.9": true, "Mozilla/5.0 (X11; U; Linux i686; hu-HU; rv:1.9.1.9) Gecko/20100330 Fedora/3.5.9-1.fc12 Firefox/3.5.9": true, @@ -176,50 +322,131 @@ var bots = map[string]bool{ "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3": true, "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1": true, "Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13": true, + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Xing Bot": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/100.0.4896.127 Safari/537.36": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/112.0.0.0 Safari/537.36": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/114.0.5735.179 Safari/537.36": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36": true, + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0(compatible;Googlebot/2.1; +http://www.google.com/bot.html)": true, + "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, + "Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true, + "Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true, + "Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true, + "Mozilla/9.0 (compatible; Staddlebot/1.0)": true, "myspider/Nutch-1.10": true, - "netEstate NE Crawler (+http://www.website-datenbank.de/)": true, - "omgili/0.5 +http://omgili.com": true, - "PercolateCrawler/4 (ops@percolate.com)": true, - "python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-43-generic": true, + "navigation": true, + "netEstate NE Crawler (+http://www.website-datenbank.de/)": true, + "Netvibes (crawler; http://www.netvibes.com)": true, + "Netvibes (crawler; https://www.netvibes.com)": true, + "Nextcloud Server Crawler": true, + "omgili/0.5 +http://omgili.com": true, + "OWLer/0.1 (built with StormCrawler; https://ows.eu/owler; owl@ow-s.eu)": true, + "PercolateCrawler/4 (ops@percolate.com)": true, + "PhxBot/0.1 (phxbot@protonmail.com)": true, + "PlayStore-Google Mozilla/5.0 Chrome/114.0.5735.179 (KHTML, like Gecko; compatible; +http://www.google.com/bot.html)": true, + "PlurkBot; Mozilla/5.0 (X11; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0(facebookexternalhitTwitterbot compatible; PlurkBot; +https://www.plurk.com/)": true, + "PubMatic Crawler Bot": true, + "PulsePoint-Ads.txt-Crawler/1.1": true, "python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-61-generic": true, - "Python-urllib/2.7": true, + "Quora-Bot/1.0 (http://www.quora.com)": true, + "R6_CommentReader(www.radian6.com/crawler)": true, + "R6_FeedFetcher(www.radian6.com/crawler)": true, + "RandomWebSiteBot 1.0": true, + "read.write.as": true, + "Readybot.io (https://readybot.io)": true, + "repology-linkchecker/1 (+https://repology.org/docs/bots)": true, "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true, "Ruby": true, - "SafeDNSBot (https://www.safedns.com/searchbot)": true, "SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)": true, + "SafeDNSBot (https://www.safedns.com/searchbot)": true, + "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; http://www.google.com/bot.html)": true, "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true, - "Screaming Frog SEO Spider/4.1": true, - "Screaming Frog SEO Spider/5.0": true, - "Screaming Frog SEO Spider/5.1": true, - "semanticbot": true, - "semanticbot (info@semanticaudience.com)": true, - "ShowyouBot (http://showyou.com/crawler)": true, - "Slackbot 1.0 (+https://api.slack.com/robots)": true, - "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)": true, - "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)": true, + "Screaming Frog SEO Spider/4.1": true, + "Screaming Frog SEO Spider/5.0": true, + "Screaming Frog SEO Spider/5.1": true, + "Sellers.Guide Crawler by Primis": true, + "semanticbot": true, + "SEMrushBot": true, + "SeRanking SEOChecker": true, + "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)": true, + "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)": true, + "ShowyouBot (http://showyou.com/crawler)": true, + "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)": true, + "Slack-ImgProxy (+https://api.slack.com/robots)": true, + "Slackbot 1.0 (+https://api.slack.com/robots)": true, + "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)": true, + "Slzji.com Search Bot (https://www.slzji.com/search_bot)": true, + "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots": true, + "Snap-URL-Preview (bot; snapchat; +https://developers.snap.com/robots)": true, + "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)": true, "spider": true, "spiderbot": true, "Stratagems Kumo": true, - "Traackr.com Bot": true, - "voltron": true, - "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, - "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true, + "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1396250262": true, + "Synapse (bot; +https://github.com/matrix-org/synapse)": true, + "TelegramBot (like TwitterBot)": true, + "TheFeedReaderBot/2.0": true, + "tiny.write.as": true, + "Traackr.com Bot": true, + "Twingly Recon": true, + "Twingly Recon-Imse/1.0 (+https://app.twingly.com/public-docs/crawler)": true, + "Twingly Recon-Wally/1.0 (+https://app.twingly.com/public-docs/crawler)": true, + "Twitterbot": true, + "voltron": true, + "webprosbot/2.0 (+mailto:abuse-6337@webpros.com)": true, + "webscraper": true, + "WhatsApp/2.23.12.78 A": true, + "Who.is Bot": true, + "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, + "wp.com feedbot/1.0 (+https://wp.com)": true, + "Write.as v1.7.0; Android": true, + "write.as": true, + "WriteFreely.org Crawler (https://writefreely.org/instances)": true, + "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true, + "yacybot (-global; amd64 Linux 5.10.0-23-amd64; java 17.0.7; Europe/en) http://yacy.net/bot.html": true, + "yacybot (/global; amd64 Linux 5.15.0-73-generic; java 1.8.0_372; Etc/en) http://yacy.net/bot.html": true, + "yacybot (/global; amd64 Windows 10 10.0; java 11.0.18; Europe/pl) http://yacy.net/bot.html": true, "yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true, + "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true, "YisouSpider": true, - "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true, + "ZoominfoBot (zoominfobot at zoominfo dot com)": true, } var botPrefixes = []string{ + "AdsTxtCrawler/", + "Akkoma ", + "Apache-HttpClient/", + "bitlybot/", + "curl/", + "DuckDuckBot/", + "facebookexternalhit/", + "FediDB/", "Friendica", + "Googlebot-Image/", + "Googlebot/", + "hackney/", + "http.rb/", + "kbinBot ", + "libwww-perl/", + "Mediumbot-MetaTagFetcher/", "Mozilla/5.0 (compatible; AhrefsBot/", "Mozilla/5.0 (compatible; Applebot/", "Mozilla/5.0 (compatible; archive.org_bot", "PHP/", + "PixelFedBot/", + "Pleroma ", + "python-requests/", + "Python-urllib/", + "RepoLookoutBot/", + "Screaming Frog SEO Spider/", + "semanticbot ", + "SummalyBot/", "TelegramBot", "Twitterbot/", - "hackney/", - "http.rb/", - "python-requests/", + "WhatsApp/", } // IsBot returns whether or not the provided User-Agent string is a known bot From 7dc8a924933d7bca0b7e5056924354384b2d3a7b Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 6 Jul 2023 18:38:34 -0400 Subject: [PATCH 67/76] Catch all Bing and Googlebots, plus clean up --- bots/bots.go | 80 ++++++++++++++++++----------------------------- bots/bots_test.go | 4 ++- 2 files changed, 33 insertions(+), 51 deletions(-) diff --git a/bots/bots.go b/bots/bots.go index 1b8d58c..e3f5a83 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -94,8 +94,6 @@ var bots = map[string]bool{ "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)": true, "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html": true, "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, - "Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, "Mozilla/5.0 (compatible; BitSightBot/1.0)": true, "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)": true, @@ -119,7 +117,6 @@ var bots = map[string]bool{ "Mozilla/5.0 (compatible; FFZBot/3.0.0; +https://www.frankerfacez.com)": true, "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true, "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true, - "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)": true, "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true, "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true, @@ -228,12 +225,8 @@ var bots = map[string]bool{ "Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone14,3__weibo__13.6.3__iphone__os15.1)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 15_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone12,8__weibo__13.6.3__iphone__os15.6)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)": true, - "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)": true, "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1": true, @@ -246,11 +239,6 @@ var bots = map[string]bool{ "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)": true, "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)": true, "Mozilla/5.0 (Linux; Android 5.1.1; A37f) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.120 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, - "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 (Linux; Android 6.0.1; vivo 1606 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36 VivoBrowser/5.7.0.6": true, "Mozilla/5.0 (Linux; Android 6.0.1; vivo Y66 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/87.0.4280.141 Mobile Safari/537.36 VivoBrowser/10.9.14.0": true, "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)": true, @@ -317,19 +305,12 @@ var bots = map[string]bool{ "Mozilla/5.0 (X11; U; Linux i686; nl; rv:1.9) Gecko/2008061015 Firefox/3.0": true, "Mozilla/5.0 (X11; U; Linux i686; pt-BR; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6": true, "Mozilla/5.0 (X11; U; Linux x86_64; cs-CZ; rv:1.9.1.7) Gecko/20100106 Ubuntu/9.10 (karmic) Firefox/3.5.7": true, - "Mozilla/5.0 (X11; U; Linux x86_64; de; rv:1.9.2.8) Googlebot-Compatible Gecko/20100723 Ubuntu/10.04 (lucid) Firefox/3.6.8": true, "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.18) Gecko/20081112 Fedora/2.0.0.18-1.fc8 Firefox/2.0.0.18": true, "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3": true, "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1": true, "Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13": true, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Xing Bot": true, "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/100.0.4896.127 Safari/537.36": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/112.0.0.0 Safari/537.36": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/114.0.5735.179 Safari/537.36": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36": true, - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0(compatible;Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true, "Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true, @@ -360,18 +341,13 @@ var bots = map[string]bool{ "repology-linkchecker/1 (+https://repology.org/docs/bots)": true, "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true, "Ruby": true, - "SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)": true, - "SafeDNSBot (https://www.safedns.com/searchbot)": true, - "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; http://www.google.com/bot.html)": true, - "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true, - "Screaming Frog SEO Spider/4.1": true, - "Screaming Frog SEO Spider/5.0": true, - "Screaming Frog SEO Spider/5.1": true, - "Sellers.Guide Crawler by Primis": true, - "semanticbot": true, - "SEMrushBot": true, - "SeRanking SEOChecker": true, - "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)": true, + "SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)": true, + "SafeDNSBot (https://www.safedns.com/searchbot)": true, + "Sellers.Guide Crawler by Primis": true, + "semanticbot": true, + "SEMrushBot": true, + "SeRanking SEOChecker": true, + "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)": true, "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)": true, "ShowyouBot (http://showyou.com/crawler)": true, "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)": true, @@ -392,25 +368,17 @@ var bots = map[string]bool{ "tiny.write.as": true, "Traackr.com Bot": true, "Twingly Recon": true, - "Twingly Recon-Imse/1.0 (+https://app.twingly.com/public-docs/crawler)": true, - "Twingly Recon-Wally/1.0 (+https://app.twingly.com/public-docs/crawler)": true, - "Twitterbot": true, - "voltron": true, - "webprosbot/2.0 (+mailto:abuse-6337@webpros.com)": true, - "webscraper": true, - "WhatsApp/2.23.12.78 A": true, - "Who.is Bot": true, - "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, - "wp.com feedbot/1.0 (+https://wp.com)": true, - "Write.as v1.7.0; Android": true, - "write.as": true, - "WriteFreely.org Crawler (https://writefreely.org/instances)": true, - "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true, - "yacybot (-global; amd64 Linux 5.10.0-23-amd64; java 17.0.7; Europe/en) http://yacy.net/bot.html": true, - "yacybot (/global; amd64 Linux 5.15.0-73-generic; java 1.8.0_372; Etc/en) http://yacy.net/bot.html": true, - "yacybot (/global; amd64 Windows 10 10.0; java 11.0.18; Europe/pl) http://yacy.net/bot.html": true, - "yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true, - "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true, + "Twitterbot": true, + "voltron": true, + "webprosbot/2.0 (+mailto:abuse-6337@webpros.com)": true, + "webscraper": true, + "Who.is Bot": true, + "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, + "wp.com feedbot/1.0 (+https://wp.com)": true, + "Write.as v1.7.0; Android": true, + "write.as": true, + "WriteFreely.org Crawler (https://writefreely.org/instances)": true, + "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true, "YisouSpider": true, "ZoominfoBot (zoominfobot at zoominfo dot com)": true, } @@ -445,8 +413,15 @@ var botPrefixes = []string{ "semanticbot ", "SummalyBot/", "TelegramBot", + "Twingly Recon-", "Twitterbot/", "WhatsApp/", + "yacybot ", +} + +var botPhrases = []string{ + "bingbot", + "Googlebot", } // IsBot returns whether or not the provided User-Agent string is a known bot @@ -463,5 +438,10 @@ func IsBot(ua string) bool { return true } } + for _, p := range botPhrases { + if strings.Contains(ua, p) { + return true + } + } return false } diff --git a/bots/bots_test.go b/bots/bots_test.go index 5c6a2ab..e2553ab 100644 --- a/bots/bots_test.go +++ b/bots/bots_test.go @@ -4,7 +4,7 @@ import "testing" func TestIsBot(t *testing.T) { tests := map[string]bool{ - "Twitterbot/1.0": true, + "Twitterbot/1.0": true, "http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true, "http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true, "http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true, @@ -13,6 +13,8 @@ func TestIsBot(t *testing.T) { "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true, "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)": true, + + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, } for ua, r := range tests { From dcd7a0cb2ca17e03046f7c71b40c710e94c46bd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:39:22 +0000 Subject: [PATCH 68/76] Bump golang.org/x/crypto from 0.0.0-20210921155107-089bfa567519 to 0.1.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.0.0-20210921155107-089bfa567519 to 0.1.0. - [Commits](https://github.com/golang/crypto/commits/v0.1.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9e9f99f..d379d9a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.7.1 github.com/writeas/slug v1.2.0 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/crypto v0.1.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect ) diff --git a/go.sum b/go.sum index d5552a3..d527048 100644 --- a/go.sum +++ b/go.sum @@ -29,13 +29,15 @@ github.com/writeas/slug v1.2.0 h1:EMQ+cwLiOcA6EtFwUgyw3Ge18x9uflUnOnR6bp/J+/g= github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQFubfQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -47,15 +49,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 5a77fdf8079e1ca5921513082999eaf55dc85a22 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 14 Sep 2023 14:26:53 -0400 Subject: [PATCH 69/76] Add "archive.org_bot" as bot phrase --- bots/bots.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bots/bots.go b/bots/bots.go index e3f5a83..5d77212 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -422,6 +422,7 @@ var botPrefixes = []string{ var botPhrases = []string{ "bingbot", "Googlebot", + "archive.org_bot", } // IsBot returns whether or not the provided User-Agent string is a known bot From e313cb0ecf1b0af37bb51ec9ad73b35a80167e7c Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 14 Sep 2023 14:27:06 -0400 Subject: [PATCH 70/76] Add Category.IsCategory field --- category/category.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/category/category.go b/category/category.go index 4ac81c3..5a131dd 100644 --- a/category/category.go +++ b/category/category.go @@ -17,6 +17,11 @@ type Category struct { Slug string `json:"slug"` Title string `json:"title"` PostCount int64 `json:"post_count"` + + // IsCategory distinguishes this Category from a mere tag. If true, it is often prominently featured on a blog, + // and looked up via its Slug instead of its Hashtag. It usually has all metadata, such as Title, correctly + // populated. + IsCategory bool `json:"-"` } // NewCategory creates a Category you can insert into the database, based on a hashtag. It automatically breaks up the From adbbafe3ee0c9693b7fc82a0b647b90fdc0b1540 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 2 Oct 2023 19:27:51 -0400 Subject: [PATCH 71/76] Add Activity.Updated field Per the Mastodon docs, we need to send an `updated` parameter for `Update` activities to work correctly. https://docs.joinmastodon.org/spec/activitypub/#supported-activities-for-statuses --- activitystreams/activity.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index ebb118f..cfdab0d 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -19,6 +19,7 @@ type Activity struct { BaseObject Actor string `json:"actor"` Published time.Time `json:"published,omitempty"` + Updated time.Time `json:"updated,omitempty"` To []string `json:"to,omitempty"` CC []string `json:"cc,omitempty"` Object *Object `json:"object"` @@ -67,6 +68,7 @@ func NewUpdateActivity(o *Object) *Activity { Actor: o.AttributedTo, Object: o, Published: o.Published, + Updated: o.Updated, } return &a } @@ -107,6 +109,7 @@ func NewFollowActivity(actorIRI, followeeIRI string) *FollowActivity { type Object struct { BaseObject Published time.Time `json:"published,omitempty"` + Updated time.Time `json:"updated,omitempty"` Summary *string `json:"summary,omitempty"` InReplyTo *string `json:"inReplyTo,omitempty"` URL string `json:"url"` From d81124d45431953dd42d7e9ce61e57a304ee0495 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Mon, 2 Oct 2023 21:30:47 -0400 Subject: [PATCH 72/76] Ensure Updated field will be omitted when nil or zero --- activitystreams/activity.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index cfdab0d..4d42a63 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -17,12 +17,12 @@ var Extensions = map[string]string{} // process of occurring, or may occur in the future. type Activity struct { BaseObject - Actor string `json:"actor"` - Published time.Time `json:"published,omitempty"` - Updated time.Time `json:"updated,omitempty"` - To []string `json:"to,omitempty"` - CC []string `json:"cc,omitempty"` - Object *Object `json:"object"` + Actor string `json:"actor"` + Published time.Time `json:"published,omitempty"` + Updated *time.Time `json:"updated,omitempty"` + To []string `json:"to,omitempty"` + CC []string `json:"cc,omitempty"` + Object *Object `json:"object"` } type FollowActivity struct { @@ -68,7 +68,9 @@ func NewUpdateActivity(o *Object) *Activity { Actor: o.AttributedTo, Object: o, Published: o.Published, - Updated: o.Updated, + } + if o.Updated != nil && !o.Updated.IsZero() { + a.Updated = o.Updated } return &a } @@ -109,7 +111,7 @@ func NewFollowActivity(actorIRI, followeeIRI string) *FollowActivity { type Object struct { BaseObject Published time.Time `json:"published,omitempty"` - Updated time.Time `json:"updated,omitempty"` + Updated *time.Time `json:"updated,omitempty"` Summary *string `json:"summary,omitempty"` InReplyTo *string `json:"inReplyTo,omitempty"` URL string `json:"url"` From ecf7fd1f90e190a6095fa9aa43fc4b1ea4b59dca Mon Sep 17 00:00:00 2001 From: Hoto Ras Date: Sun, 31 Mar 2024 13:42:18 +0900 Subject: [PATCH 73/76] Temporal addition of Korean l10n Since the localization project didn't allow me to contribute, I ended up open ip the repo, add up the file, localize, and finally have it able to apply. - Added: l10n/phrases_ko.go // korean l10n - Modified: l10n/strings.go // Add casing to have above change able to apply --- l10n/phrases_ko.go | 34 ++++++++++++++++++++++++++++++++++ l10n/strings.go | 2 ++ 2 files changed, 36 insertions(+) create mode 100644 l10n/phrases_ko.go diff --git a/l10n/phrases_ko.go b/l10n/phrases_ko.go new file mode 100644 index 0000000..b253732 --- /dev/null +++ b/l10n/phrases_ko.go @@ -0,0 +1,34 @@ +package l10n + +// Temporal korean localation by Hoto Ras (@ras@hoto.moe), 2024 +var phrasesKO = map[string]string{ + "Anonymous post": "모두의 게시물", + "Blogs": "블로그", + "Enter": "접속", + "Newer": "신규", + "Next": "다음", + "Older": "오래됨", + "Posts": "게시물", + "Previous": "이전", + "Publish to...": "다음으로 게시...", + "Publish": "게시", + "Read more...": "더 읽어보기...", + "Subscribe": "구독", + "This blog requires a password.": "이 블로그는 비밀번호가 필요합니다.", + "Toggle theme": "테마 토글", + "View posts": "게시물 보기", + "delete": "삭제", + "edit": "수정", + "email subscription confirm": "이메일을 확인하고 확인 링크를 눌러 구독을 완료하세요.", + "email subscription prompt": "새로운 소식을 보고 싶으신가요? 여기에 이메일을 입력해 구독하세요!", + "email subscription success": "구독이 완료되었습니다! 이제 앞으로의 블로그 게시물을 이메일로 받아볼 수 있어요.", + "move to...": "다음으로 이동...", + "pin": "고정", + "published with write.as": "write.as를 통해 배포됨", + "share modal ending": "친구에게 보내거나, 인터넷에 공유하거나, 트윗할 수도 있습니다. 더 알아보기", + "share modal introduction": "각각의 게시물에는 비밀이 있어요. 각자 특별한 주소가 있어 모두와 공유할 수 있답니다. 주소는 여기 있어요:", + "share modal title": "이 게시물 공유하기", + "share": "공유", + "unpin": "고정 해제", + "title dash": "—", +} diff --git a/l10n/strings.go b/l10n/strings.go index 5481f29..e3d0f55 100644 --- a/l10n/strings.go +++ b/l10n/strings.go @@ -34,6 +34,8 @@ func Strings(lang string) map[string]string { return phrasesIT case "ja": return phrasesJA + case "ko": + return phrasesKO case "lt": return phrasesLT case "mk": From e8804848c7e0736fc62109dbb63b06e52d188f2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:45:43 +0000 Subject: [PATCH 74/76] Bump golang.org/x/crypto from 0.1.0 to 0.17.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.1.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.1.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index d379d9a..6c4ea5a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.7.1 github.com/writeas/slug v1.2.0 - golang.org/x/crypto v0.1.0 + golang.org/x/crypto v0.17.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect ) diff --git a/go.sum b/go.sum index d527048..945d92a 100644 --- a/go.sum +++ b/go.sum @@ -30,17 +30,17 @@ github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQF github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -49,20 +49,23 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From 428972d76c55ea0134dec1223d37b152a351cb0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 May 2025 20:23:00 +0000 Subject: [PATCH 75/76] Bump golang.org/x/crypto from 0.17.0 to 0.35.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.17.0 to 0.35.0. - [Commits](https://github.com/golang/crypto/compare/v0.17.0...v0.35.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.35.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6c4ea5a..c23c1f7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/writeas/openssl-go v1.0.0 github.com/writeas/saturday v1.7.1 github.com/writeas/slug v1.2.0 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.35.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect ) diff --git a/go.sum b/go.sum index 945d92a..b7f3fea 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuP github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -30,20 +31,33 @@ github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQF github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -52,24 +66,36 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From cf9f5db56bbc1277ace87be0ea116e594f4f93ba Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Wed, 7 May 2025 17:52:29 -0400 Subject: [PATCH 76/76] Add Preview field to Object This supports the `preview` AS2 attribute to support short-form previews for `article`s, per FEP-b2b8. --- activitystreams/activity.go | 1 + 1 file changed, 1 insertion(+) diff --git a/activitystreams/activity.go b/activitystreams/activity.go index 4d42a63..915ae56 100644 --- a/activitystreams/activity.go +++ b/activitystreams/activity.go @@ -123,6 +123,7 @@ type Object struct { ContentMap map[string]string `json:"contentMap,omitempty"` Tag []Tag `json:"tag,omitempty"` Attachment []Attachment `json:"attachment,omitempty"` + Preview *Object `json:"preview,omitempty"` // Person Inbox string `json:"inbox,omitempty"`