From ad95da63b7ec78a389c211d208ddaecead3fae4e Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sat, 26 Sep 2020 20:21:16 -0700 Subject: [PATCH] Use local geo database instead of npm package. --- .gitignore | 2 +- Dockerfile | 3 +++ lib/request.js | 10 +++------ package.json | 13 ++++++------ scripts/build-geo.js | 49 ++++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 12 ++--------- 6 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 scripts/build-geo.js diff --git a/.gitignore b/.gitignore index ca0f3c4f7c..68408a6668 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ /build /public/umami.js /lang-compiled -/lang-formatted +/geo # misc .DS_Store diff --git a/Dockerfile b/Dockerfile index 31ea0054d9..a70d7f8713 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,9 @@ COPY --from=build /build/prod_node_modules ./node_modules # Copy generated Prisma client COPY --from=build /build/node_modules/.prisma/ ./node_modules/.prisma/ +# Copy geo database +COPY --from=build /build/geo ./geo + COPY --from=build /build/yarn.lock /build/package.json ./ COPY --from=build /build/.next ./.next COPY --from=build /build/public ./public diff --git a/lib/request.js b/lib/request.js index 35658ce5d6..6975c170dc 100644 --- a/lib/request.js +++ b/lib/request.js @@ -1,8 +1,8 @@ +import path from 'path'; import requestIp from 'request-ip'; import { browserName, detectOS } from 'detect-browser'; import isLocalhost from 'is-localhost-ip'; import maxmind from 'maxmind'; -import geolite2 from 'geolite2-redist'; import { DESKTOP_OS, MOBILE_OS, @@ -60,15 +60,11 @@ export async function getCountry(req, ip) { } // Database lookup - const lookup = await geolite2.open('GeoLite2-Country', path => { - return maxmind.open(path); - }); + const lookup = await maxmind.open(path.resolve(__dirname, '../geo/GeoLite2-Country.mmdb')); const result = lookup.get(ip); - lookup.close(); - - return result.country.iso_code; + return result?.country?.iso_code; } export async function getClientInfo(req, { screen }) { diff --git a/package.json b/package.json index 107d7dab58..d615260ded 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "umami", - "version": "0.56.0", + "version": "0.57.0", "description": "A simple, fast, website analytics alternative to Google Analytics. ", "author": "Mike Cao ", "license": "MIT", @@ -11,19 +11,20 @@ }, "scripts": { "dev": "next dev", - "build": "npm-run-all build-tracker build-lang build-db build-app", + "build": "npm-run-all build-tracker build-lang build-geo build-db build-app", "start": "next start", "build-app": "next build", "build-tracker": "rollup -c rollup.tracker.config.js", "build-db": "npm-run-all copy-db-schema build-db-client", "build-lang": "npm-run-all format-lang compile-lang", - "copy-db-schema": "node scripts/copy-db-schema.js", + "build-geo": "node scripts/build-geo.js", "build-db-schema": "dotenv prisma introspect", "build-db-client": "dotenv prisma generate", "build-mysql-schema": "dotenv prisma introspect -- --schema=./prisma/schema.mysql.prisma", "build-mysql-client": "dotenv prisma generate -- --schema=./prisma/schema.mysql.prisma", "build-postgresql-schema": "dotenv prisma introspect -- --schema=./prisma/schema.postgresql.prisma", "build-postgresql-client": "dotenv prisma generate -- --schema=./prisma/schema.postgresql.prisma", + "copy-db-schema": "node scripts/copy-db-schema.js", "generate-lang": "npm-run-all extract-lang merge-lang", "extract-lang": "formatjs extract {pages,components}/**/*.js --out-file build/messages.json", "merge-lang": "node scripts/merge-lang.js", @@ -60,9 +61,7 @@ "date-fns": "^2.16.1", "date-fns-tz": "^1.0.10", "detect-browser": "^5.1.1", - "dotenv": "^8.2.0", "formik": "^2.1.5", - "geolite2-redist": "^1.0.7", "immer": "^7.0.9", "is-localhost-ip": "^1.4.0", "isbot-fast": "^1.2.0", @@ -95,6 +94,7 @@ "@svgr/webpack": "^5.4.0", "cross-env": "^7.0.2", "del": "^5.1.0", + "dotenv": "^8.2.0", "dotenv-cli": "^4.0.0", "eslint": "^7.9.0", "eslint-config-prettier": "^6.11.0", @@ -116,6 +116,7 @@ "stylelint": "^13.7.1", "stylelint-config-css-modules": "^2.2.0", "stylelint-config-prettier": "^8.0.1", - "stylelint-config-recommended": "^3.0.0" + "stylelint-config-recommended": "^3.0.0", + "tar": "^6.0.5" } } diff --git a/scripts/build-geo.js b/scripts/build-geo.js new file mode 100644 index 0000000000..dc17397eab --- /dev/null +++ b/scripts/build-geo.js @@ -0,0 +1,49 @@ +require('dotenv').config(); +const fs = require('fs'); +const path = require('path'); +const https = require('https'); +const zlib = require('zlib'); +const tar = require('tar'); + +let url = + 'https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/GeoLite2-Country.tar.gz'; + +if (process.env.MAXMIND_LICENSE_KEY) { + url = + `https://download.maxmind.com/app/geoip_download` + + `?edition_id=GeoLite2-Country&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`; +} + +const dest = path.resolve(__dirname, '../geo'); + +if (!fs.existsSync(dest)) { + fs.mkdirSync(dest); +} + +const download = url => + new Promise(resolve => { + https.get(url, res => { + resolve(res.pipe(zlib.createGunzip({})).pipe(tar.t())); + }); + }); + +download(url).then( + res => + new Promise((resolve, reject) => { + res.on('entry', entry => { + if (entry.path.endsWith('.mmdb')) { + const filename = path.join(dest, path.basename(entry.path)); + entry.pipe(fs.createWriteStream(filename)); + + console.log('Saved geo database:', filename); + } + }); + + res.on('error', e => { + reject(e); + }); + res.on('finish', () => { + resolve(); + }); + }), +); diff --git a/yarn.lock b/yarn.lock index 136b534d64..d659c87b32 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4260,14 +4260,6 @@ gensync@^1.0.0-beta.1: resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.1.tgz#58f4361ff987e5ff6e1e7a210827aa371eaac269" integrity sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg== -geolite2-redist@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/geolite2-redist/-/geolite2-redist-1.0.7.tgz#98ecd0260115a7c90bc0d49fa96e04f8ab56cddb" - integrity sha512-NrsPDYUU7OVTtZzj5McnBI7b2n8teS+zgX3IXSd3qxF9M2OaDA4SeF1loWzDyzkQzPFr8JHEYYIBTD0sQGQ7ug== - dependencies: - rimraf "^3.0.2" - tar "^6.0.2" - get-own-enumerable-property-symbols@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz#b5fde77f22cbe35f390b4e089922c50bce6ef664" @@ -7637,7 +7629,7 @@ rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.3, rimraf@^2.7.1: dependencies: glob "^7.1.3" -rimraf@^3.0.0, rimraf@^3.0.2: +rimraf@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -8543,7 +8535,7 @@ tar@^4.4.2: safe-buffer "^5.1.2" yallist "^3.0.3" -tar@^6.0.2: +tar@^6.0.5: version "6.0.5" resolved "https://registry.yarnpkg.com/tar/-/tar-6.0.5.tgz#bde815086e10b39f1dcd298e89d596e1535e200f" integrity sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg==