Merge pull request #387 from microsoft/sarveshn/identity

Improve identify API interface

lerna.json

@@ -2,7 +2,7 @@
   "packages": [
     "packages/*"
   ],
-  "version": "0.7.7",
+  "version": "0.7.8",
   "npmClient": "yarn",
   "useWorkspaces": true
 }

package.json

@@ -1,7 +1,7 @@
 {
   "name": "clarity",
   "private": true,
-  "version": "0.7.7",
+  "version": "0.7.8",
   "repository": "https://github.com/microsoft/clarity.git",
   "author": "Sarvesh Nagpal <sarveshn@microsoft.com>",
   "license": "MIT",

packages/clarity-decode/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-decode",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",
@@ -26,7 +26,7 @@
     "url": "https://github.com/Microsoft/clarity/issues"
   },
   "dependencies": {
-    "clarity-js": "^0.7.7"
+    "clarity-js": "^0.7.8"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^24.0.0",

packages/clarity-devtools/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-devtools",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "private": true,
   "description": "Adds Clarity debugging support to browser devtools",
   "author": "Microsoft Corp.",
@@ -24,9 +24,9 @@
     "url": "https://github.com/Microsoft/clarity/issues"
   },
   "dependencies": {
-    "clarity-decode": "^0.7.7",
-    "clarity-js": "^0.7.7",
-    "clarity-visualize": "^0.7.7"
+    "clarity-decode": "^0.7.8",
+    "clarity-js": "^0.7.8",
+    "clarity-visualize": "^0.7.8"
   },
   "devDependencies": {
     "@rollup/plugin-node-resolve": "^15.0.0",

packages/clarity-devtools/static/manifest.json

@@ -2,8 +2,8 @@
   "manifest_version": 2,
   "name": "Microsoft Clarity Developer Tools",
   "description": "Clarity helps you understand how users are interacting with your website.",
-  "version": "0.7.7",
-  "version_name": "0.7.7",
+  "version": "0.7.8",
+  "version_name": "0.7.8",
   "minimum_chrome_version": "50",
   "devtools_page": "devtools.html",
   "icons": {
@@ -43,4 +43,4 @@
     "storage",
     "tabs"
   ]
-}
+}

packages/clarity-js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-js",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",
@@ -8,6 +8,7 @@
   "module": "build/clarity.module.js",
   "unpkg": "build/clarity.min.js",
   "insight": "build/clarity.insight.js",
+  "performance": "build/clarity.performance.js",
   "types": "types/index.d.ts",
   "keywords": [
     "clarity",

packages/clarity-js/rollup.config.ts

@@ -58,5 +58,27 @@ export default [
       terser({output: {comments: false}}),
       commonjs({ include: ["node_modules/**"] })
     ]
+  },
+  {
+    input: "src/global.ts",
+    output: [ { file: pkg.performance, format: "iife", exports: "named" } ],
+    onwarn(message, warn) {
+      if (message.code === 'CIRCULAR_DEPENDENCY') { return; }
+      warn(message);
+    },
+    plugins: [
+      alias({
+        entries: [
+          { find: /@src\/interaction.*/, replacement: '@src/performance/blank' },
+          { find: /@src\/layout.*/, replacement: '@src/performance/blank' },
+          { find: /@src\/diagnostic.*/, replacement: '@src/performance/blank' },
+          { find: /@src\/data\/(extract|baseline|summary)/, replacement: '@src/performance/blank' }
+        ]
+      }),
+      resolve(),
+      typescript(),
+      terser({output: {comments: false}}),
+      commonjs({ include: ["node_modules/**"] })
+    ]
   }
 ];

packages/clarity-js/src/core/config.ts

@@ -17,7 +17,8 @@ let config: Config = {
     upload: null,
     fallback: null,
     upgrade: null,
-    action: null
+    action: null,
+    dob: null
 };
 
 export default config;

packages/clarity-js/src/core/scrub.ts

@@ -64,7 +64,7 @@ export function text(value: string, hint: string, privacy: Privacy, mangle: bool
                 switch (hint) {
                     case Layout.Constant.TextTag:
                     case Layout.Constant.DataAttribute:
-                        return scrub(value);
+                        return scrub(value, Data.Constant.Letter, Data.Constant.Digit);
                     case "value":
                     case "input":
                     case "click":
@@ -83,7 +83,10 @@ export function text(value: string, hint: string, privacy: Privacy, mangle: bool
     return value;
 }
 
-export function url(input: string): string {
+export function url(input: string, electron: boolean = false): string {
+    // Replace the URL for Electron apps so we don't send back file:/// URL
+    if (electron) { return `${Data.Constant.HTTPS}${Data.Constant.Electron}`; }
+
     let drop = config.drop;
     if (drop && drop.length > 0 && input && input.indexOf("?") > 0) {
       let [path, query] = input.split("?");
@@ -109,9 +112,9 @@ function mask(value: string): string {
     return value.replace(catchallRegex, Data.Constant.Mask);
 }
 
-function scrub(value: string): string {
+export function scrub(value: string, letter: string, digit: string): string {
     regex(); // Initialize regular expressions
-    return value.replace(letterRegex, Data.Constant.Letter).replace(digitRegex, Data.Constant.Digit);
+    return value ? value.replace(letterRegex, letter).replace(digitRegex, digit) : value;
 }
 
 function mangleToken(value: string): string {
@@ -161,7 +164,7 @@ function redact(value: string): string {
                 // Check if unicode regex is supported, otherwise fallback to calling mask function on this token
                 if (unicodeRegex && currencyRegex !== null) {
                     // Do not redact information if the token contains a currency symbol
-                    token = token.match(currencyRegex) ? token : token.replace(letterRegex, Data.Constant.Letter).replace(digitRegex, Data.Constant.Digit);
+                    token = token.match(currencyRegex) ? token : scrub(token, Data.Constant.Letter, Data.Constant.Digit);
                 } else {
                     token = mask(token);
                 }

packages/clarity-js/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.7.7";
+let version = "0.7.8";
 export default version;

packages/clarity-js/src/data/extract.ts

@@ -81,7 +81,7 @@ export function compute(): void {
                 let selectorKey = parseInt(s);
                 let nodes = document.querySelectorAll(selectorData[selectorKey]) as NodeListOf<HTMLElement>;
                 if (nodes) {
-                    let text = Array.from(nodes).map(e => e.innerText)
+                    let text = Array.from(nodes).map(e => e.textContent)
                     update(key, selectorKey, text.join(Constant.Seperator).substring(0, Setting.ExtractLimit));
                 }
             }

packages/clarity-js/src/data/limit.ts

@@ -15,6 +15,7 @@ export function check(bytes: number): void {
     if (data.check === Check.None) {
         let reason = data.check;
         reason = envelope.data.sequence >= Setting.PayloadLimit ? Check.Payload : reason;
+        reason = envelope.data.pageNum >= Setting.PageLimit ? Check.Page : reason;
         reason = time() > Setting.ShutdownLimit ? Check.Shutdown : reason;
         reason = bytes > Setting.PlaybackBytesLimit ? Check.Shutdown : reason;
         if (reason !== data.check) {

packages/clarity-js/src/data/metadata.ts

@@ -16,6 +16,7 @@ export function start(): void {
   rootDomain = null;
   const ua = navigator && "userAgent" in navigator ? navigator.userAgent : Constant.Empty;
   const title = document && document.title ? document.title : Constant.Empty;
+  const electron = ua.indexOf(Constant.Electron) > 0 ? BooleanFlag.True : BooleanFlag.False;
 
   // Populate ids for this page
   let s = session();
@@ -26,20 +27,23 @@ export function start(): void {
   // Override configuration based on what's in the session storage, unless it is blank (e.g. using upload callback, like in devtools)
   config.lean = config.track && s.upgrade !== null ? s.upgrade === BooleanFlag.False : config.lean;
   config.upload = config.track && typeof config.upload === Constant.String && s.upload && s.upload.length > Constant.HTTPS.length ? s.upload : config.upload;
-  
+
   // Log page metadata as dimensions
   dimension.log(Dimension.UserAgent, ua);
   dimension.log(Dimension.PageTitle, title);
-  dimension.log(Dimension.Url, scrub.url(location.href));
+  dimension.log(Dimension.Url, scrub.url(location.href, !!electron));
   dimension.log(Dimension.Referrer, document.referrer);
   dimension.log(Dimension.TabId, tab());
   dimension.log(Dimension.PageLanguage, document.documentElement.lang);
   dimension.log(Dimension.DocumentDirection, document.dir);
   dimension.log(Dimension.DevicePixelRatio, `${window.devicePixelRatio}`);
-  
+  dimension.log(Dimension.Dob, u.dob.toString());
+  dimension.log(Dimension.CookieVersion, u.version.toString());
+
   // Capture additional metadata as metrics
   metric.max(Metric.ClientTimestamp, s.ts);
-  metric.max(Metric.Playback, BooleanFlag.False); 
+  metric.max(Metric.Playback, BooleanFlag.False);
+  metric.max(Metric.Electron, electron);
 
   // Capture navigator specific dimensions
   if (navigator) {
@@ -48,7 +52,7 @@ export function start(): void {
     metric.max(Metric.MaxTouchPoints, navigator.maxTouchPoints);
     metric.max(Metric.DeviceMemory, Math.round((<any>navigator).deviceMemory));
     userAgentData();
-  } 
+  }
 
   if (screen) {
     metric.max(Metric.ScreenWidth, Math.round(screen.width));
@@ -69,12 +73,12 @@ export function start(): void {
 function userAgentData(): void {
   let uaData = navigator["userAgentData"];
   if (uaData && uaData.getHighEntropyValues) {
-    uaData.getHighEntropyValues(["model","platform","platformVersion","uaFullVersion"]).then(ua => { 
-      dimension.log(Dimension.Platform, ua.platform); 
-      dimension.log(Dimension.PlatformVersion, ua.platformVersion); 
+    uaData.getHighEntropyValues(["model","platform","platformVersion","uaFullVersion"]).then(ua => {
+      dimension.log(Dimension.Platform, ua.platform);
+      dimension.log(Dimension.PlatformVersion, ua.platformVersion);
       ua.brands?.forEach(brand => { dimension.log(Dimension.Brand, brand.name + Constant.Tilde + brand.version); });
-      dimension.log(Dimension.Model, ua.model); 
-      metric.max(Metric.Mobile, ua.mobile ? BooleanFlag.True : BooleanFlag.False); 
+      dimension.log(Dimension.Model, ua.model);
+      metric.max(Metric.Mobile, ua.mobile ? BooleanFlag.True : BooleanFlag.False);
     });
   } else { dimension.log(Dimension.Platform, navigator.platform); }
 }
@@ -149,9 +153,13 @@ function track(u: User, consent: BooleanFlag = null): void {
   // Convert time precision into days to reduce number of bytes we have to write in a cookie
   // E.g. Math.ceil(1628735962643 / (24*60*60*1000)) => 18852 (days) => ejo in base36 (13 bytes => 3 bytes)
   let end = Math.ceil((Date.now() + (Setting.Expire * Time.Day))/Time.Day);
+  // If DOB is not set in the user object, use the date set in the config as a DOB
+  let dob = u.dob === 0 ? (config.dob === null ? 0 : config.dob) : u.dob;
+
   // To avoid cookie churn, write user id cookie only once every day
-  if (u.expiry === null || Math.abs(end - u.expiry) >= Setting.CookieInterval || u.consent !== consent) {
-    setCookie(Constant.CookieKey, [data.userId, Setting.CookieVersion, end.toString(36), consent].join(Constant.Pipe), Setting.Expire);
+  if (u.expiry === null || Math.abs(end - u.expiry) >= Setting.CookieInterval || u.consent !== consent || u.dob !== dob) {
+    let cookieParts = [data.userId, Setting.CookieVersion, end.toString(36), consent, dob];
+    setCookie(Constant.CookieKey, cookieParts.join(Constant.Pipe), Setting.Expire);
   }
 }
 
@@ -185,7 +193,7 @@ function num(string: string, base: number = 10): number {
 }
 
 function user(): User {
-  let output: User = { id: shortid(), expiry: null, consent: BooleanFlag.False };
+  let output: User = { id: shortid(), version: 0, expiry: null, consent: BooleanFlag.False, dob: 0 };
   let cookie = getCookie(Constant.CookieKey);
   if(cookie && cookie.length > 0) {
     // Splitting and looking up first part for forward compatibility, in case we wish to store additional information in a cookie
@@ -205,9 +213,11 @@ function user(): User {
     }
     // End code for backward compatibility
     // Read version information and timestamp from cookie, if available
+    if (parts.length > 1) { output.version = num(parts[1]); }
     if (parts.length > 2) { output.expiry = num(parts[2], 36); }
     // Check if we have explicit consent to track this user
     if (parts.length > 3 && num(parts[3]) === 1) { output.consent = BooleanFlag.True; }
+    if (parts.length > 4 && num(parts[1]) > 1) { output.dob = num(parts[4]); }
     // Set track configuration to true for this user if we have explicit consent, regardless of project setting
     config.track = config.track || output.consent === BooleanFlag.True;
     // Get user id from cookie only if we tracking is enabled, otherwise fallback to a random id
@@ -246,7 +256,7 @@ function setCookie(key: string, value: string, time: number): void {
           rootDomain = `.${hostname[i]}${rootDomain ? rootDomain : Constant.Empty}`;
           // We do not wish to attempt writing a cookie on the absolute last part of the domain, e.g. .com or .net.
           // So we start attempting after second-last part, e.g. .domain.com (PASS) or .co.uk (FAIL)
-          if (i < hostname.length - 1) { 
+          if (i < hostname.length - 1) {
             // Write the cookie on the current computed top level domain
             document.cookie = `${cookie}${Constant.Semicolon}${Constant.Domain}${rootDomain}`;
             // Once written, check if the cookie exists and its value matches exactly with what we intended to set

packages/clarity-js/src/data/variable.ts

@@ -1,5 +1,6 @@
-import { Constant, Event, VariableData } from "@clarity-types/data";
+import { Constant, Event, IdentityData, Setting, VariableData } from "@clarity-types/data";
 import * as core from "@src/core";
+import { scrub } from "@src/core/scrub";
 import encode from "./encode";
 
 export let data: VariableData = null;
@@ -13,10 +14,28 @@ export function set(variable: string, value: string | string[]): void {
     log(variable, values);
 }
 
-export function identify(userId: string, sessionId: string = null, pageId: string = null): void {
-    log(Constant.UserId, [userId]);
-    log(Constant.SessionId, [sessionId]);
-    log(Constant.PageId, [pageId]);
+export async function identify(userId: string, sessionId: string = null, pageId: string = null, userHint: string = null): Promise<IdentityData> {
+    let output: IdentityData = { userId: await sha256(userId), userHint: userHint || redact(userId) };
+
+    // By default, hash custom userId using SHA256 algorithm on the client to preserve privacy
+    log(Constant.UserId, [output.userId]);
+    
+    // Optional non-identifying name for the user
+    // If name is not explicitly provided, we automatically generate a redacted version of the userId
+    log(Constant.UserHint, [output.userHint]);
+    log(Constant.UserType, [detect(userId)]);    
+
+    // Log sessionId and pageId if provided
+    if (sessionId) { 
+        log(Constant.SessionId, [sessionId]);
+        output.sessionId = sessionId;
+    }
+    if (pageId) {
+        log(Constant.PageId, [pageId]);
+        output.pageId = pageId;
+    }
+
+    return output;
 }
 
 function log(variable: string, value: string[]): void {
@@ -44,3 +63,22 @@ export function reset(): void {
 export function stop(): void {
     reset();
 }
+
+function redact(input: string): string {
+    return input && input.length >= Setting.WordLength ? 
+        `${input.substring(0,2)}${scrub(input.substring(2), Constant.Asterix, Constant.Asterix)}` : scrub(input, Constant.Asterix, Constant.Asterix);
+}
+
+async function sha256(input: string): Promise<string> {
+    try {
+        if (crypto && input) {
+            // Reference: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest#converting_a_digest_to_a_hex_string
+            const buffer = await crypto.subtle.digest(Constant.SHA256, new TextEncoder().encode(input));
+            return Array.prototype.map.call(new Uint8Array(buffer), (x: any) =>(('00'+x.toString(16)).slice(-2))).join('');
+        } else { return Constant.Empty; }
+    } catch { return Constant.Empty; }
+}
+
+function detect(input: string): string {
+    return input && input.indexOf(Constant.At) > 0 ? Constant.Email : Constant.String;
+}

packages/clarity-js/src/interaction/click.ts

@@ -92,10 +92,10 @@ function text(element: Node): string {
         // Grab text using "textContent" for most HTMLElements, however, use "value" for HTMLInputElements and "alt" for HTMLImageElement.
         let t = element.textContent || (element as HTMLInputElement).value || (element as HTMLImageElement).alt;
         if (t) {
-            // Trim any spaces at the beginning or at the end of string
-            // Also, replace multiple occurrence of space characters with a single white space
+            // Replace multiple occurrence of space characters with a single white space
+            // Also, trim any spaces at the beginning or at the end of string
             // Finally, send only first few characters as specified by the Setting
-            output = t.trim().replace(/\s+/g, Constant.Space).substr(0, Setting.ClickText);
+            output = t.replace(/\s+/g, Constant.Space).trim().substr(0, Setting.ClickText);
         }
     }
     return output;

packages/clarity-js/src/performance/blank.ts

@@ -0,0 +1,7 @@
+export * from "@src/insight/blank";
+
+export let keys = [];
+
+/* Intentionally blank module with empty code */
+export function hashText(): void {}
+export function trigger(): void {}

packages/clarity-js/types/core.d.ts

@@ -39,7 +39,7 @@ export const enum ExtractSource {
 }
 
 export const enum Type {
-    Array = 1, 
+    Array = 1,
     Object = 2,
     Simple = 3
 }
@@ -136,6 +136,7 @@ export interface Config {
     fallback?: string;
     upgrade?: (key: string) => void;
     action?: (key: string) => void;
+    dob?: number;
 }
 
 export const enum Constant {

packages/clarity-js/types/data.d.ts

@@ -5,7 +5,7 @@ export type DecodedToken = (any | any[]);
 
 export type MetadataCallback = (data: Metadata, playback: boolean) => void;
 export interface MetadataCallbackOptions {
-    callback: MetadataCallback, 
+    callback: MetadataCallback,
     wait: boolean
 }
 
@@ -105,7 +105,8 @@ export const enum Metric {
     Iframed = 31,
     MaxTouchPoints = 32,
     HardwareConcurrency = 33,
-    DeviceMemory = 34
+    DeviceMemory = 34,
+    Electron = 35
 }
 
 export const enum Dimension {
@@ -136,7 +137,9 @@ export const enum Dimension {
     Brand = 24,
     Model = 25,
     DevicePixelRatio = 26,
-    ConnectionType = 27
+    ConnectionType = 27,
+    Dob = 28,
+    CookieVersion = 29
 }
 
 export const enum Check {
@@ -146,7 +149,8 @@ export const enum Check {
     Retry = 3,
     Bytes = 4,
     Collection = 5,
-    Server = 6
+    Server = 6,
+    Page = 7
 }
 
 export const enum Code {
@@ -190,7 +194,7 @@ export const enum IframeStatus {
 export const enum Setting {
     Expire = 365, // 1 Year
     SessionExpire = 1, // 1 Day
-    CookieVersion = 1, // Increment this version every time there's a cookie schema change
+    CookieVersion = 2, // Increment this version every time there's a cookie schema change
     SessionTimeout = 30 * Time.Minute, // 30 minutes
     CookieInterval = 1, // 1 Day
     PingInterval = 1 * Time.Minute, // 1 Minute
@@ -198,6 +202,7 @@ export const enum Setting {
     SummaryInterval = 100, // Same events within 100ms will be collapsed into single summary
     ClickText = 25, // Maximum number of characters to send as part of Click event's text field
     PayloadLimit = 128, // Do not allow more than specified payloads per page
+    PageLimit = 128, // Do not allow more than 128 pages in a session
     ShutdownLimit = 2 * Time.Hour, // Shutdown instrumentation after specified time
     RetryLimit = 1, // Maximum number of attempts to upload a payload before giving up
     PlaybackBytesLimit = 10 * 1024 * 1024, // 10MB
@@ -218,7 +223,7 @@ export const enum Setting {
     MinUploadDelay = 100, // Minimum time before we are ready to flush events to the server
     MaxUploadDelay = 30 * Time.Second, // Do flush out payload once every 30s,
     ExtractLimit = 10000, // Do not extract more than 10000 characters
-    ChecksumPrecision = 24, // n-bit integer to represent token hash 
+    ChecksumPrecision = 24, // n-bit integer to represent token hash
     UploadTimeout = 15000 // Timeout in ms for XHR requests
 }
 
@@ -249,6 +254,8 @@ export const enum Constant {
     Dropped = "*na*",
     Comma = ",",
     Dot = ".",
+    At = "@",
+    Asterix = "*",
     Semicolon = ";",
     Equals = "=",
     Path = ";path=/",
@@ -258,6 +265,7 @@ export const enum Constant {
     Top = "_top",
     String = "string",
     Number = "number",
+    Email = "email",
     CookieKey = "_clck", // Clarity Cookie Key
     SessionKey = "_clsk", // Clarity Session Key
     TabKey = "_cltk", // Clarity Tab Key
@@ -266,6 +274,8 @@ export const enum Constant {
     Upgrade = "UPGRADE",
     Action = "ACTION",
     Extract = "EXTRACT",
+    UserHint = "userHint",
+    UserType = "userType",
     UserId = "userId",
     SessionId = "sessionId",
     PageId = "pageId",
@@ -290,7 +300,9 @@ export const enum Constant {
     ConditionEnd = "}",
     Seperator = "<SEP>",
     Timeout = "Timeout",
-    Bang = "!"
+    Bang = "!",
+    SHA256 = "SHA-256",
+    Electron = "Electron"
 }
 
 export const enum XMLReadyState {
@@ -332,8 +344,10 @@ export interface Session {
 
 export interface User {
     id: string;
+    version: number;
     expiry: number;
     consent: BooleanFlag;
+    dob: number;
 }
 
 export interface Envelope extends Metadata {
@@ -372,6 +386,13 @@ export interface BaselineData {
     activityTime: number;
 }
 
+export interface IdentityData {
+    userId: string;
+    userHint: string;
+    sessionId?: string;
+    pageId?: string;
+}
+
 export interface DimensionData {
     [key: number]: string[];
 }

packages/clarity-js/types/index.d.ts

@@ -14,7 +14,7 @@ interface Clarity {
   consent: () => void;
   event: (name: string, value: string) => void;
   set: (variable: string, value: string | string[]) => void;
-  identify: (userId: string, sessionId?: string, pageId?: string) => void;
+  identify: (userId: string, sessionId?: string, pageId?: string, userHint?: string) => void;
   metadata: (callback: Data.MetadataCallback, wait?: boolean) => void;
 }
 

packages/clarity-visualize/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-visualize",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",
@@ -27,7 +27,7 @@
     "url": "https://github.com/Microsoft/clarity/issues"
   },
   "dependencies": {
-    "clarity-decode": "^0.7.7"
+    "clarity-decode": "^0.7.8"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^24.0.0",