More compact cookie storage with the option to use local storage (amplitude#244)

Cookies had this fancy abstraction to mimic local storage. This led to
excessively large cookies.

Rather than storing a large JSON serialized version of the cookie data we use
positional values.

Base64 encoding also led to considerable cookie bloat. Instead we only
Base64 encode the user id field and assume other fields in the cookie
are safe.

You can also forgo using cookie storage altogether with the
disableCookie option. This prevents tracking users across different
subdomains of your site.

Author: Krishna Rajendran

src/amplitude-client.js

@@ -1,5 +1,6 @@
 import Constants from './constants';
 import cookieStorage from './cookiestorage';
+import MetadataStorage from './metaDataStorage';
 import getUtmData from './utm';
 import Identify from './identify';
 import localStorage from './localstorage';  // jshint ignore:line
@@ -32,7 +33,6 @@ if (BUILD_COMPAT_REACT_NATIVE) {
  */
 var AmplitudeClient = function AmplitudeClient(instanceName) {
   this._instanceName = utils.isEmptyString(instanceName) ? Constants.DEFAULT_INSTANCE : instanceName.toLowerCase();
-  this._legacyStorageSuffix = this._instanceName === Constants.DEFAULT_INSTANCE ? '' : '_' + this._instanceName;
   this._unsentEvents = [];
   this._unsentIdentifys = [];
   this._ua = new UAParser(navigator.userAgent).getResult();
@@ -76,37 +76,63 @@ AmplitudeClient.prototype.init = function init(apiKey, opt_userId, opt_config, o
   }
 
   try {
-    this.options.apiKey = apiKey;
-    this._storageSuffix = '_' + apiKey + this._legacyStorageSuffix;
+    _parseConfig(this.options, opt_config);
 
-    var hasExistingCookie = !!this.cookieStorage.get(this.options.cookieName + this._storageSuffix);
-    if (opt_config && opt_config.deferInitialization && !hasExistingCookie) {
-      this._deferInitialization(apiKey, opt_userId, opt_config, opt_callback);
-      return;
+    if (this.options.cookieName !== DEFAULT_OPTIONS.cookieName) {
+      utils.log.warn('The cookieName option is deprecated. We will be ignoring it for newer cookies');
     }
 
-    _parseConfig(this.options, opt_config);
+    this.options.apiKey = apiKey;
+    this._storageSuffix = '_' + apiKey + (this._instanceName === Constants.DEFAULT_INSTANCE ? '' : '_' + this._instanceName);
+    this._storageSuffixV5 = apiKey.slice(0,6);
 
-    if (type(this.options.logLevel) === 'string') {
-      utils.setLogLevel(this.options.logLevel);
-    }
+    this._oldCookiename = this.options.cookieName + this._storageSuffix;
+    this._unsentKey = this.options.unsentKey + this._storageSuffix;
+    this._unsentIdentifyKey = this.options.unsentIdentifyKey + this._storageSuffix;
 
-    var trackingOptions = _generateApiPropertiesTrackingConfig(this);
-    this._apiPropertiesTrackingOptions = Object.keys(trackingOptions).length > 0 ? {tracking_options: trackingOptions} : {};
+    this._cookieName = Constants.COOKIE_PREFIX + '_' + this._storageSuffixV5;
 
     this.cookieStorage.options({
       expirationDays: this.options.cookieExpiration,
       domain: this.options.domain,
       secure: this.options.secureCookie,
       sameSite: this.options.sameSiteCookie
     });
+
+    this._metadataStorage = new MetadataStorage({
+      storageKey: this._cookieName,
+      disableCookies: this.options.disableCookies,
+      expirationDays: this.options.cookieExpiration,
+      domain: this.options.domain,
+      secure: this.options.secureCookie,
+      sameSite: this.options.sameSiteCookie
+    });
+
+    const hasOldCookie = !!this.cookieStorage.get(this._oldCookiename);
+    const hasNewCookie = !!this._metadataStorage.load();
+    this._useOldCookie = (!hasNewCookie && hasOldCookie) && !this.options.cookieForceUpgrade;
+    const hasCookie = hasNewCookie || hasOldCookie;
     this.options.domain = this.cookieStorage.options().domain;
 
-    if (!BUILD_COMPAT_REACT_NATIVE) {
-      if (this._instanceName === Constants.DEFAULT_INSTANCE) {
+    if (this.options.deferInitialization && !hasCookie) {
+      this._deferInitialization(apiKey, opt_userId, opt_config, opt_callback);
+      return;
+    }
+
+    if (type(this.options.logLevel) === 'string') {
+      utils.setLogLevel(this.options.logLevel);
+    }
+
+    var trackingOptions = _generateApiPropertiesTrackingConfig(this);
+    this._apiPropertiesTrackingOptions = Object.keys(trackingOptions).length > 0 ? {tracking_options: trackingOptions} : {};
+
+    if (this.options.cookieForceUpgrade && hasOldCookie) {
+      if (!hasNewCookie) {
         _upgradeCookieData(this);
       }
+      this.cookieStorage.remove(this._oldCookiename);
     }
+
     _loadCookieData(this);
     this._pendingReadStorage = true;
 
@@ -508,75 +534,31 @@ AmplitudeClient.prototype._setInStorage = function _setInStorage(storage, key, v
   storage.setItem(key + this._storageSuffix, value);
 };
 
-var _upgradeCookieData = function _upgradeCookieData(scope) {
-  // skip if already migrated to 4.10+
-  var cookieData = scope.cookieStorage.get(scope.options.cookieName + scope._storageSuffix);
-  if (type(cookieData) === 'object') {
-    return;
-  }
-  // skip if already migrated to 2.70+
-  cookieData = scope.cookieStorage.get(scope.options.cookieName + scope._legacyStorageSuffix);
-  if (type(cookieData) === 'object' && cookieData.deviceId && cookieData.sessionId && cookieData.lastEventTime) {
-    return;
-  }
-
-  var _getAndRemoveFromLocalStorage = function _getAndRemoveFromLocalStorage(key) {
-    var value = localStorage.getItem(key);
-    localStorage.removeItem(key);
-    return value;
-  };
-
-  // in v2.6.0, deviceId, userId, optOut was migrated to localStorage with keys + first 6 char of apiKey
-  var apiKeySuffix = (type(scope.options.apiKey) === 'string' && ('_' + scope.options.apiKey.slice(0, 6))) || '';
-  var localStorageDeviceId = _getAndRemoveFromLocalStorage(Constants.DEVICE_ID + apiKeySuffix);
-  var localStorageUserId = _getAndRemoveFromLocalStorage(Constants.USER_ID + apiKeySuffix);
-  var localStorageOptOut = _getAndRemoveFromLocalStorage(Constants.OPT_OUT + apiKeySuffix);
-  if (localStorageOptOut !== null && localStorageOptOut !== undefined) {
-    localStorageOptOut = String(localStorageOptOut) === 'true'; // convert to boolean
-  }
-
-  // pre-v2.7.0 event and session meta-data was stored in localStorage. move to cookie for sub-domain support
-  var localStorageSessionId = parseInt(_getAndRemoveFromLocalStorage(Constants.SESSION_ID));
-  var localStorageLastEventTime = parseInt(_getAndRemoveFromLocalStorage(Constants.LAST_EVENT_TIME));
-  var localStorageEventId = parseInt(_getAndRemoveFromLocalStorage(Constants.LAST_EVENT_ID));
-  var localStorageIdentifyId = parseInt(_getAndRemoveFromLocalStorage(Constants.LAST_IDENTIFY_ID));
-  var localStorageSequenceNumber = parseInt(_getAndRemoveFromLocalStorage(Constants.LAST_SEQUENCE_NUMBER));
-
-  var _getFromCookie = function _getFromCookie(key) {
-    return type(cookieData) === 'object' && cookieData[key];
-  };
-  scope.options.deviceId = _getFromCookie('deviceId') || localStorageDeviceId;
-  scope.options.userId = _getFromCookie('userId') || localStorageUserId;
-  scope._sessionId = _getFromCookie('sessionId') || localStorageSessionId || scope._sessionId;
-  scope._lastEventTime = _getFromCookie('lastEventTime') || localStorageLastEventTime || scope._lastEventTime;
-  scope._eventId = _getFromCookie('eventId') || localStorageEventId || scope._eventId;
-  scope._identifyId = _getFromCookie('identifyId') || localStorageIdentifyId || scope._identifyId;
-  scope._sequenceNumber = _getFromCookie('sequenceNumber') || localStorageSequenceNumber || scope._sequenceNumber;
-
-  // optOut is a little trickier since it is a boolean
-  scope.options.optOut = localStorageOptOut || false;
-  if (cookieData && cookieData.optOut !== undefined && cookieData.optOut !== null) {
-    scope.options.optOut = String(cookieData.optOut) === 'true';
-  }
-
-  _saveCookieData(scope);
-};
-
 /**
  * Fetches deviceId, userId, event meta data from amplitude cookie
  * @private
  */
 var _loadCookieData = function _loadCookieData(scope) {
-  var cookieData = scope.cookieStorage.get(scope.options.cookieName + scope._storageSuffix);
+  if (!scope._useOldCookie) {
+    const props = scope._metadataStorage.load();
+    if (type(props) === 'object') {
+      _loadCookieDataProps(scope, props);
+    }
+    return;
+  }
 
+  var cookieData = scope.cookieStorage.get(scope._oldCookiename);
   if (type(cookieData) === 'object') {
     _loadCookieDataProps(scope, cookieData);
-  } else {
-    var legacyCookieData = scope.cookieStorage.get(scope.options.cookieName + scope._legacyStorageSuffix);
-    if (type(legacyCookieData) === 'object') {
-      scope.cookieStorage.remove(scope.options.cookieName + scope._legacyStorageSuffix);
-      _loadCookieDataProps(scope, legacyCookieData);
-    }
+    return;
+  }
+};
+
+const _upgradeCookieData = (scope) => {
+  var cookieData = scope.cookieStorage.get(scope._oldCookiename);
+  if (type(cookieData) === 'object') {
+    _loadCookieDataProps(scope, cookieData);
+    _saveCookieData(scope);
   }
 };
 
@@ -594,19 +576,19 @@ var _loadCookieDataProps = function _loadCookieDataProps(scope, cookieData) {
     }
   }
   if (cookieData.sessionId) {
-    scope._sessionId = parseInt(cookieData.sessionId);
+    scope._sessionId = parseInt(cookieData.sessionId, 10);
   }
   if (cookieData.lastEventTime) {
-    scope._lastEventTime = parseInt(cookieData.lastEventTime);
+    scope._lastEventTime = parseInt(cookieData.lastEventTime, 10);
   }
   if (cookieData.eventId) {
-    scope._eventId = parseInt(cookieData.eventId);
+    scope._eventId = parseInt(cookieData.eventId, 10);
   }
   if (cookieData.identifyId) {
-    scope._identifyId = parseInt(cookieData.identifyId);
+    scope._identifyId = parseInt(cookieData.identifyId, 10);
   }
   if (cookieData.sequenceNumber) {
-    scope._sequenceNumber = parseInt(cookieData.sequenceNumber);
+    scope._sequenceNumber = parseInt(cookieData.sequenceNumber, 10);
   }
 };
 
@@ -628,7 +610,12 @@ var _saveCookieData = function _saveCookieData(scope) {
   if (AsyncStorage) {
     AsyncStorage.setItem(scope._storageSuffix, JSON.stringify(cookieData));
   }
-  scope.cookieStorage.set(scope.options.cookieName + scope._storageSuffix, cookieData);
+
+  if (scope._useOldCookie) {
+    scope.cookieStorage.set(scope.options.cookieName + scope._storageSuffix, cookieData);
+  } else {
+    scope._metadataStorage.save(cookieData);
+  }
 };
 
 /**

src/constants.js

@@ -19,6 +19,7 @@ export default {
   USER_ID: 'amplitude_userId',
 
   COOKIE_TEST: 'amplitude_cookie_test',
+  COOKIE_PREFIX: "amp",
 
   // revenue keys
   REVENUE_EVENT: 'revenue_amount',

src/cookie.js

@@ -3,6 +3,7 @@
  */
 
 import Base64 from './base64';
+import Constants from './constants';
 import utils from './utils';
 import getLocation from './get-location';
 import baseCookie from './base-cookie';
@@ -27,7 +28,12 @@ const getHost = (url) => {
   return a.hostname || location.hostname; 
 };
 
+let _topDomain = '';
+
 const topDomain = (url) => {
+  if (_topDomain) {
+    return topDomain;
+  }
   const host = getHost(url);
   const parts = host.split('.');
   const last = parts[parts.length - 1];
@@ -121,6 +127,20 @@ var set = function(name, value) {
   }
 };
 
+var setRaw = function(name, value) {
+  try {
+    baseCookie.set(_domainSpecific(name), value, _options);
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
+
+var getRaw = function(name) {
+  var nameEq = _domainSpecific(name) + '=';
+  return baseCookie.get(nameEq);
+};
+
 
 var remove = function(name) {
   try {
@@ -131,10 +151,34 @@ var remove = function(name) {
   }
 };
 
+let _areCookiesEnabled = null;
+
+// test that cookies are enabled - navigator.cookiesEnabled yields false positives in IE, need to test directly
+const areCookiesEnabled = () => {
+  if (_areCookiesEnabled !== null) {
+    return _areCookiesEnabled;
+  }
+  var uid = String(new Date());
+  var result;
+  try {
+    set(Constants.COOKIE_TEST, uid);
+    _areCookiesEnabled = get(Constants.COOKIE_TEST) === uid;
+    remove(Constants.COOKIE_TEST);
+    return result;
+  } catch (e) {
+    // cookies are not enabled
+  }
+  return false;
+};
+
 export default {
   reset,
   options,
+  topDomain,
   get,
   set,
-  remove
+  remove,
+  areCookiesEnabled,
+  setRaw,
+  getRaw
 };

src/metadataStorage.js

@@ -0,0 +1,78 @@
+/*
+ * Persist SDK event metadata
+ * Uses cookie if available, otherwise fallback to localstorage.
+ */
+
+import Base64 from './base64';
+import Cookie from './cookie';
+import baseCookie from './base-cookie';
+import localStorage from './localstorage'; // jshint ignore:line
+
+class MetadataStorage {
+  constructor({storageKey, disableCookies, domain, secure, sameSite, expirationDays}) {
+    this.storageKey = storageKey;
+    this.disableCookieStorage = !Cookie.areCookiesEnabled() || disableCookies;
+    this.domain = domain;
+    this.secure = secure;
+    this.sameSite = sameSite;
+    this.expirationDays = expirationDays;
+    this.topDomain = domain || Cookie.topDomain();
+  }
+
+  getCookieStorageKey() {
+    return `${this.storageKey}${this.domain ? `_${this.domain}` : ''}`;
+  }
+
+  save({ deviceId, userId, optOut, sessionId, lastEventTime, eventId, identifyId, sequenceNumber }) {
+    // do not change the order of these items
+    const value = `${deviceId}.${Base64.encode(userId || '')}.${optOut ? '1' : ''}.${sessionId}.${lastEventTime}.${eventId}.${identifyId}.${sequenceNumber}`;
+
+    if (this.disableCookieStorage) {
+      localStorage.setItem(this.storageKey, value);
+    } else {
+      baseCookie.set(
+        this.getCookieStorageKey(),
+        value,
+        { domain: this.topDomain, secure: this.secure, sameSite: this.sameSite, expirationDays: this.expirationDays }
+      );
+    }
+  }
+
+  load() {
+    let str;
+    if (!this.disableCookieStorage) {
+      str = baseCookie.get(this.getCookieStorageKey() + '=');
+    }
+    if (!str) {
+      str = localStorage.getItem(this.storageKey);
+    }
+
+    if (!str) {
+      return null;
+    }
+
+    const values = str.split('.');
+
+    let userId = null;
+    if (values[1]) {
+      try {
+        userId = Base64.decode(values[1]);
+      } catch (e) {
+        userId = null;
+      }
+    }
+
+    return {
+      deviceId: values[0],
+      userId,
+      optOut: values[2] === '1',
+      sessionId: parseInt(values[3], 10),
+      lastEventTime: parseInt(values[4], 10),
+      eventId: parseInt(values[5], 10),
+      identifyId: parseInt(values[6], 10),
+      sequenceNumber: parseInt(values[7], 10)
+    };
+  }
+}
+
+export default MetadataStorage;