diff --git a/src/annotator/guest.js b/src/annotator/guest.js
index ee4c12d519c..29042b63267 100644
--- a/src/annotator/guest.js
+++ b/src/annotator/guest.js
@@ -476,7 +476,7 @@ export default class Guest extends Delegator {
 
       const highlights = /** @type {AnnotationHighlight[]} */ (highlightRange(
         range,
-        Math.random() * (6 - 1) + 1
+        annotation.$color
       ));
       highlights.forEach(h => {
         h._annotation = anchor.annotation;
diff --git a/src/annotator/highlighter.js b/src/annotator/highlighter.js
index 8a80bfd2dc1..d785d151f56 100644
--- a/src/annotator/highlighter.js
+++ b/src/annotator/highlighter.js
@@ -212,13 +212,13 @@ function wholeTextNodesInRange(range) {
  * element of the specified class and returns the highlight Elements.
  *
  * @param {Range} range - Range to be highlighted
- * @param {Number} colorNum - number to index color list
+ * @param {String} color - What color to display the annotation in
  * @param {string} cssClass - A CSS class to use for the highlight
  * @return {HighlightElement[]} - Elements wrapping text in `normedRange` to add a highlight effect
  */
 export function highlightRange(
   range,
-  colorNum,
+  color,
   cssClass = 'hypothesis-highlight'
 ) {
   const textNodes = wholeTextNodesInRange(range);
@@ -265,11 +265,7 @@ export function highlightRange(
     /** @type {HighlightElement} */
     const highlightEl = document.createElement('hypothesis-highlight');
     highlightEl.className = cssClass;
-    //here is where we can call another function to get the color for the user
-    //color-->index number in the 'user-color' class
-    if (colorNum !== -1) {
-      highlightEl.classList.toggle('user-color' + Math.round(colorNum), true);
-    }
+    highlightEl.style.backgroundColor = color;
 
     nodes[0].parentNode.replaceChild(highlightEl, nodes[0]);
     nodes.forEach(node => highlightEl.appendChild(node));
@@ -361,6 +357,7 @@ export function setHighlightsFocused(highlights, focused) {
 export function setHighlightsVisible(root, visible) {
   const showHighlightsClass = 'hypothesis-highlights-always-on';
   root.classList.toggle(showHighlightsClass, visible);
+  root.classList.toggle('hypothesis-highlights-hidden', !visible);
 }
 
 /**
diff --git a/src/sidebar/components/SidebarView.js b/src/sidebar/components/SidebarView.js
index bdf3861de02..36f1bd71e29 100644
--- a/src/sidebar/components/SidebarView.js
+++ b/src/sidebar/components/SidebarView.js
@@ -56,7 +56,6 @@ function SidebarView({
   const searchUris = store.searchUris();
   const sidebarHasOpened = store.hasSidebarOpened();
   const userId = store.profile().userid;
-  //WILLNOTE we can use this to hide/show for this user!
 
   // The local `$tag` of a direct-linked annotation; populated once it
   // has anchored: meaning that it's ready to be focused and scrolled to
diff --git a/src/sidebar/services/frame-sync.js b/src/sidebar/services/frame-sync.js
index 1fe30830296..12d76a83aef 100644
--- a/src/sidebar/services/frame-sync.js
+++ b/src/sidebar/services/frame-sync.js
@@ -14,12 +14,20 @@ import { watch } from '../util/watch';
  * within the current session and anchor it in the document.
  */
 export function formatAnnot(ann) {
+  function hashColor(s) {
+    let h = 0;
+    for (let i = 0; i < s.length; i++) {
+      h = (Math.imul(31, h) + s.charCodeAt(i)) | 0;
+    }
+    return `hsl(${Math.abs(h) % 360}, 100%, 85%)`;
+  }
   return {
     tag: ann.$tag,
     msg: {
       document: ann.document,
       target: ann.target,
       uri: ann.uri,
+      $color: hashColor(ann.user),
     },
   };
 }
diff --git a/src/sidebar/services/test/frame-sync-test.js b/src/sidebar/services/test/frame-sync-test.js
index ff4454ec0b9..bd05509a073 100644
--- a/src/sidebar/services/test/frame-sync-test.js
+++ b/src/sidebar/services/test/frame-sync-test.js
@@ -233,7 +233,7 @@ describe('sidebar/services/frame-sync', function () {
       });
 
       it('should not create an annotation in the sidebar', () => {
-        const ann = { target: [] };
+        const ann = { target: [], user: 'acct:test@hypothes.is' };
 
         fakeBridge.emit('beforeCreateAnnotation', { tag: 't1', msg: ann });
 
@@ -241,21 +241,21 @@ describe('sidebar/services/frame-sync', function () {
       });
 
       it('should open the sidebar', () => {
-        const ann = { target: [] };
+        const ann = { target: [], user: 'acct:test@hypothes.is' };
         fakeBridge.emit('beforeCreateAnnotation', { tag: 't1', msg: ann });
 
         assert.calledWith(fakeBridge.call, 'showSidebar');
       });
 
       it('should open the login prompt panel', () => {
-        const ann = { target: [] };
+        const ann = { target: [], user: 'acct:test@hypothes.is' };
         fakeBridge.emit('beforeCreateAnnotation', { tag: 't1', msg: ann });
 
         assert.calledWith(fakeStore.openSidebarPanel, 'loginPrompt');
       });
 
       it('should send a "deleteAnnotation" message to the frame', () => {
-        const ann = { target: [] };
+        const ann = { target: [], user: 'acct:test@hypothes.is' };
         fakeBridge.emit('beforeCreateAnnotation', { tag: 't1', msg: ann });
 
         assert.calledWith(fakeBridge.call, 'deleteAnnotation');
diff --git a/src/sidebar/store/modules/annotations.js b/src/sidebar/store/modules/annotations.js
index bec238be220..b78a7bac899 100644
--- a/src/sidebar/store/modules/annotations.js
+++ b/src/sidebar/store/modules/annotations.js
@@ -86,7 +86,6 @@ function initializeAnnotation(annotation, tag) {
     $anchorTimeout: false,
     $tag: annotation.$tag || tag,
     $orphan: orphan,
-    $doodle: annotation.$doodle,
   });
 }
 
diff --git a/src/styles/annotator/highlights.scss b/src/styles/annotator/highlights.scss
index e999e514292..4108e70cc53 100644
--- a/src/styles/annotator/highlights.scss
+++ b/src/styles/annotator/highlights.scss
@@ -6,6 +6,12 @@
   fill: transparent;
 }
 
+.hypothesis-highlights-hidden {
+  .hypothesis-highlight {
+    background-color: transparent !important;
+  }
+}
+
 // `hypothesis-highlights-always-on` is a class that is toggled on the root
 // of the annotated document when highlights are enabled/disabled.
 .hypothesis-highlights-always-on {
diff --git a/src/types/annotator.js b/src/types/annotator.js
index 83d39d7d399..6d005818238 100644
--- a/src/types/annotator.js
+++ b/src/types/annotator.js
@@ -31,6 +31,7 @@
  * @prop {string} uri
  * @prop {Target[]} target
  * @prop {string} $tag
+ * @prop {string} $color - What color to display the highlight in? This corresponds to the user who created the annotation.
  * @prop {boolean} [$doodle] -
  *   Flag indicating that this annotation is a doodle
  * @prop {DoodleLine[]} [doodleLines] -