Localization config (OriginProtocol#414)

* Localization config

* Test wrap text

* changes

* Translation locale select

Co-authored-by: Nick Poulden <[email protected]>

TRANSLATION.md

@@ -0,0 +1,8 @@
+# Translation
+Note: Most of the code related to translation is (shamelessly) copied over from [origin repo](https://github.com/originprotocol/origin).
+
+## Adding a new language
+
+1. Update `shop/scripts/crowdinToFbt.js` file, line number 9, to add the new locale to the `locales` array
+2. Create a new file `shop/translation/crowding/all-messages_{{NEW_LOCALE_HERE}}.json` with the contents of `shop/translation/crowding/all-messages.json` 
+3. Run `npm run translate`

crowdin.yml

@@ -0,0 +1,4 @@
+files:
+  - source: /shop/translation/crowdin/all-messages.json
+    translation: >-
+      /shop/translation/crowdin/all-messages_%locale_with_underscore%.json

shop/.gitignore

@@ -9,4 +9,9 @@ public/*
 backend/scripts/output*
 backend/dist
 TODO
-.env*
+.env*
+
+/.enum_manifest.json
+/.src_manifest.json
+/.source_strings.json
+/.translated_fbts.json

shop/package.json

@@ -8,40 +8,47 @@
     "build:js": "NODE_ENV=production webpack --loglevel notice",
     "build:css": "node scripts/getCss > public/app.css",
     "build:dist": "rm -rf ../backend/dist && NODE_ENV=production npm run build && cp -r public ../backend/dist",
-    "build": "npm run build:css && npm run build:js",
+    "build": "npm run build:css && npm run build:js && npm run translate",
     "lint": "eslint . && npm run prettier:check",
     "prettier": "prettier --write *.js \"{src,test}/**/*.js\"",
     "prettier:check": "prettier -c *.js \"src/**/*.js\"",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "fbt:manifest": "node -r @babel/register ../node_modules/babel-plugin-fbt/bin/manifest --src src",
+    "fbt:collect": "node -r @babel/register ../node_modules/babel-plugin-fbt/bin/collectFBT --manifest --pretty < .src_manifest.json > .source_strings.json",
+    "fbt:translate": "node  -r @babel/register ../node_modules/babel-plugin-fbt/bin/translate.js --translations translation/fbt/*.json --jenkins --pretty > .translated_fbts.json",
+    "fbt:clean": "rm .enum_manifest.json .src_manifest.json .source_strings.json .translated_fbts.json translation/fbt/*.json 2&> /dev/null || exit 0",
+    "translate": "npm run fbt:manifest && npm run fbt:collect && node scripts/fbtToCrowdin && node scripts/crowdinToFbt && npm run fbt:translate && node scripts/splitTranslations && cp .enum_manifest.json translation/fbt/.enum_manifest.json",
+    "translate:proof": "npm run fbt:manifest && npm run fbt:collect && node scripts/fbtToCrowdin && node scripts/crowdinToFbt proof && npm run fbt:translate && node scripts/splitTranslations && cp .enum_manifest.json translation/fbt/.enum_manifest.json"
   },
   "author": "Nick Poulden <[email protected]>",
   "license": "MIT",
   "dependencies": {
-    "@babel/core": "^7.10.4",
+    "@babel/core": "^7.11.1",
     "@babel/plugin-transform-flow-strip-types": "^7.10.4",
     "@babel/preset-react": "^7.10.4",
-    "@babel/register": "^7.10.4",
+    "@babel/register": "^7.10.5",
     "@origin/contracts": "^0.8.6",
     "@origin/ipfs": "^0.1.0",
     "@origin/services": "^0.1.0",
     "@origin/utils": "^0.1.0",
     "@popperjs/core": "^2.4.4",
     "@uphold/uphold-sdk-javascript": "^2.4.0",
     "babel-plugin-module-resolver": "^4.0.0",
-    "blueimp-load-image": "^5.13.0",
+    "blueimp-load-image": "^5.14.0",
     "chartist": "^0.11.4",
     "ckeditor4-react": "^1.1.1",
-    "dayjs": "^1.8.29",
+    "dayjs": "^1.8.33",
     "dotenv": "^8.2.0",
     "ethers": "5.0.8",
+    "fbt-runtime": "^0.9.4",
     "flexsearch": "^0.6.32",
-    "openpgp": "^4.10.4",
+    "openpgp": "^4.10.7",
     "prettier": "^2.0.5",
     "query-string": "^6.13.1",
     "randomstring": "^1.1.5",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
-    "react-image-crop": "^8.6.4",
+    "react-image-crop": "^8.6.5",
     "react-select": "^3.1.0",
     "react-spring": "^8.0.27",
     "react-stripe-elements": "^6.1.2",
@@ -95,8 +102,8 @@
     "@babel/runtime": "7.11.2",
     "babel-eslint": "10.1.0",
     "babel-loader": "8.1.0",
-    "babel-plugin-fbt": "0.15.0",
-    "babel-plugin-fbt-runtime": "0.9.12",
+    "babel-plugin-fbt": "^0.15.1-beta",
+    "babel-plugin-fbt-runtime": "^0.9.12",
     "bootstrap": "4.5.2",
     "clean-webpack-plugin": "3.0.0",
     "css-loader": "4.2.1",

shop/scripts/crowdinToFbt.js

@@ -0,0 +1,138 @@
+// Iterates over files in `./translation/crowdin` over our languages, expecting them to be in key-value format that crowdin uses
+// Output: files in fbt formatin in `./translation/fbt` dir
+
+const doTestMark = process.argv.length>=3 && process.argv[2]=='proof'
+if (doTestMark) {
+  console.warn('⚠️ Proofread mode: Doing translation with test marks included for testing.')
+}
+
+const locales = [
+  'en_US', 
+  'es_ES',
+  'id_ID',
+  'ko_KR',
+  'ru_RU',
+  'vi_VN',
+  'zh_CN',
+  'zh_TW'
+]
+
+// Decodes variable names from crowdin into their original name.
+// See encoding format in fbtToCrowdin.js
+const EqualPrefix = 'E_'
+const VarPrefix = 'VAR_'
+const b64Prefix = '_B64_'
+
+function b64Decode(data) {
+  // The "/" and "+" characters cause MT to alter
+  // the data so they were replace during encoding.
+  const b64Encoded = data
+    .replace(/SLASH/g, '/')
+    .replace(/PLUS/g, '+')
+  return new Buffer(b64Encoded, 'base64').toString()
+}
+
+function decodeVarName(encodedVarName) {
+  let data = encodedVarName
+  let originalVarName = ''
+
+  if (data.startsWith(EqualPrefix)) {
+    data = data.slice(EqualPrefix.length)
+    originalVarName += '='
+  }
+
+  if (!data.startsWith(VarPrefix)) {
+    throw new Error(`Unexpected variable format. Missing prefix ${VarPrefix}. Var=${encodedVarName}`)
+  }
+  data = data.slice(VarPrefix.length)
+
+  // Extract the parts from the encoded variable name.
+  const parts = data.split(b64Prefix)
+  if (parts.length !== 2) {
+    throw new Error(`Unexpected variable format. Expected 2 parts. Var=${encodedVarName}`)
+  }
+
+  // Base64 decode the variable name.
+  const b64 = parts[1]
+  originalVarName += b64Decode(b64)
+
+  return originalVarName
+}
+
+
+function decode(str) {
+  // Special case where the entire string was just concatenated variables.
+  if (str.startsWith('{' + VarPrefix) && str.endsWith('}') && !str.includes('_B64_')) {
+    const b64 = str.slice(1 + VarPrefix.length, str.length -1)
+    const decodedStr = b64Decode(b64)
+    return '{' + decodedStr + '}'
+  }
+
+  let out=''
+  let encodedVarName = ''
+  let inBracket = false
+  for (let i = 0; i < str.length; i++) {
+    const cur = str.charAt(i)
+    if (cur==='{') {
+      inBracket=true
+      continue
+    } else if (cur==='}') {
+      inBracket=false
+      const decodedVarName = decodeVarName(encodedVarName)
+      out += '{' + decodedVarName  + '}'
+      encodedVarName = ''
+      continue
+    }
+    if (inBracket) {
+      encodedVarName += cur
+    } else {
+      out += cur
+    }
+  }
+  return out
+}
+
+
+locales.forEach(locale => {
+  // If testing, we use English for all
+  const srcFile = doTestMark ?
+    `${__dirname}/../translation/crowdin/all-messages.json` :
+    `${__dirname}/../translation/crowdin/all-messages_${locale}.json`
+  const dstFile = `${__dirname}/../translation/fbt/${locale}.json`
+
+  const fs = require('fs')
+  const translations = {}
+
+  let stringKeyValue=''
+  try {
+    stringKeyValue = JSON.parse(fs.readFileSync(srcFile))
+  }
+  catch (error) {
+    console.warn(`Could not find or parse file: ${srcFile}`)
+    return
+  }
+  console.log(`Processing file: ${srcFile}`)
+
+  Object.keys(stringKeyValue).forEach(key => {
+    const val = doTestMark ? '◀'+decode(stringKeyValue[key])+'▶' : decode(stringKeyValue[key])
+    translations[key] = {
+      'translations': [
+        { 'translation': val }
+      ]
+    }
+  })
+
+
+  const file = {
+    'fb-locale': locale,
+    'translations': translations
+  }
+
+  const output = JSON.stringify(file, null, 2)
+
+  // Write out fbt translation format
+  fs.writeFileSync(dstFile, output)
+
+  console.log(`✅ ${locale}`)
+
+})

shop/scripts/fbtToCrowdin.js

@@ -0,0 +1,108 @@
+// Input: Source strings file from fbt, json in fbt format
+// Output: Simple key-value json of strings, for consumption by Crowdin
+
+const srcFile = `${__dirname}/../.source_strings.json`
+const dstFile = `${__dirname}/../translation/crowdin/all-messages.json`
+
+const fs = require('fs')
+
+const facebookTranslations = fs.readFileSync(srcFile)
+const phrases = JSON.parse(facebookTranslations).phrases
+const allMessages = {}
+
+// To prevent machine translation from translating variables,
+// we encode them into an untranslatable string composed of several parts:
+//  - Optional "E_" if the argument has a "=" prefix.
+//  - VAR_ prefix
+//  - Variable name, with all letters in upper case and spaces replaced with "_".
+//    The variable name is to help giving extra context to the translators.
+//  - Base64 encoded version of the original variable name, with no "=" padding at the end.
+//    This is to be able to use the original name for converting language
+//    files exported by crowdin back to fbt format. Prefixed by "_B64_".
+//
+// For example, the variable {=Apple and banana} gets converted into
+// {E_VAR_APPLE_AND_BANANA_B64_QXBwbGUgYW5kIGJhbmFuYQ}
+//
+// For the decoding counterpart of this method, see crowdinToFbt.js
+
+const EqualPrefix = 'E_'
+const VarPrefix = 'VAR_'
+const b64Prefix = '_B64_'
+
+function b64Encode(str) {
+  return new Buffer.from(str).toString('base64')
+    .replace(/=/g, '')       // Strip base64 padding. It is not essential.
+    .replace(/\//g, 'SLASH')  // Replace '/' since otherwise MT alters the string.
+    .replace(/\+/g, 'PLUS')   // Replace '+' since otherwise MT alters the string.
+}
+
+function encodeVarName(varName) {
+  let name = varName
+
+  let prefix = ''
+  if (varName.startsWith('=')) {
+    prefix = EqualPrefix
+    name = name.slice(1)
+  }
+
+  const sanitizedName = name
+    .replace(/ /g, '_') // Replace spaces with underscores
+    .replace(/[^a-zA-Z0-9_]/g, '') // Strip any non alphabet character
+    .toUpperCase() // Convert to upper case.
+
+  // Base 64 encode the original variable name and sanitize it
+  // so that MT does not try to alter it.
+  const b64Name = b64Encode(name)
+
+  return prefix + VarPrefix + sanitizedName + b64Prefix + b64Name
+}
+
+function encode(str) {
+  // Special case for strings that are just a concatenation of variables.
+  // For ex.: "{=var1}{=var2}...{=var3}"
+  // This happens when a blob of html is wrapped into a fbt tag, such as:
+  // </fbt desc="blob">
+  //   This is
+  //   <b> a huge</b>
+  //   blob
+  // </fbt>
+  // In that case we base64 encode the whole string.
+  if (str.startsWith('{=') && str.endsWith('}')) {
+    const b64 = b64Encode(str.slice(1, str.length-1))
+    return '{' + VarPrefix + b64 + '}'
+  }
+
+  let out = ''
+  let varName = ''
+  let inBracket = false
+  for (let i = 0; i < str.length; i++) {
+    const cur = str.charAt(i)
+    if (cur === '{') {
+      inBracket=true
+      continue
+    } else if (cur === '}') {
+      inBracket=false
+      const encodedVarName = encodeVarName(varName)
+      out += '{' + encodedVarName + '}'
+      varName = ''
+      continue
+    }
+    if (inBracket) {
+      varName += cur
+    } else {
+      out += cur
+    }
+  }
+  return out
+}
+
+phrases.forEach(phrase => {
+  Object.keys(phrase.hashToText)
+    .forEach(key => {
+      allMessages[key] = encode(phrase.hashToText[key])
+    })
+
+})
+
+const output = JSON.stringify(allMessages, null, 2)
+fs.writeFileSync(dstFile, output)

shop/scripts/splitTranslations.js

@@ -0,0 +1,13 @@
+const fs = require('fs')
+
+const rawTranslations = fs.readFileSync(`${__dirname}/../.translated_fbts.json`)
+const translations = JSON.parse(rawTranslations)
+const translationsDir = `${__dirname}/../public/translations`
+
+fs.mkdirSync(translationsDir, { recursive: true })
+Object.keys(translations).forEach((lang) => {
+  fs.writeFileSync(
+    `${translationsDir}/${lang}.json`,
+    JSON.stringify(translations[lang], null, 2)
+  )
+})

shop/src/components/LocaleSelect.js

@@ -0,0 +1,37 @@
+import React from 'react'
+
+import { useStateValue } from 'data/state'
+import Languages from 'data/Languages'
+
+const LocaleSelect = () => {
+  const [{ locale }, dispatch] = useStateValue()
+
+  const onChange = (e) => {
+    const locale = e.target.value
+    dispatch({
+      type: 'setLocale',
+      locale
+    })
+  }
+
+  return (
+    <div className="currency-select">
+      <select
+        className="form-control currency-select"
+        value={locale}
+        onChange={onChange}
+      >
+        {Languages.map(([value, label]) => (
+          <option key={value} value={value}>
+            {label}
+          </option>
+        ))}
+      </select>
+    </div>
+  )
+}
+
+export default LocaleSelect
+
+require('react-styl')(`
+`)