Send CQ alert emails to the tree sheriffs

This CL enables sending alert emails to the tree sheriffs. As part of
the process, functions that retrieve sheriff's email address are moved
to tree_status.py.

BUG=None
TEST=`cbuildbot/run_tests`

Change-Id: Ib166d7d6e6e40bb9c7df1955d4e3c0a7d1f6a470
Reviewed-on: https://chromium-review.googlesource.com/212883
Tested-by: Yu-Ju Hong <yjhong@chromium.org>
Reviewed-by: Don Garrett <dgarrett@chromium.org>
Commit-Queue: Yu-Ju Hong <yjhong@chromium.org>

9 files changed, 111 insertions, 79 deletions

diff --git a/cbuildbot/cbuildbot_config.py b/cbuildbot/cbuildbot_config.py
index b008235ff..179d70249 100755
--- a/cbuildbot/cbuildbot_config.py
+++ b/cbuildbot/cbuildbot_config.py
@@ -294,7 +294,12 @@ _settings = dict(
   health_threshold=0,
 
 # health_alert_recipients -- List of email addresses to send health alerts to
-#                            for this builder.
+#                            for this builder. It supports automatic email
+#                            adderss lookup for the following sheriff types:
+#                                'tree': tree sheriffs
+#                                'build': build deputies
+#                                'lab': lab sheriffs
+#                                'chrome': chrome gardeners
   health_alert_recipients=[],
 
 # internal -- Whether this is an internal build config.
@@ -1653,7 +1658,8 @@ internal_paladin.add_config('pre-cq-launcher',
   manifest_version=False,
   # Every Pre-CQ launch failure should send out an alert.
   health_threshold=1,
-  health_alert_recipients=['chromeos-build-alerts@google.com'],
+  health_alert_recipients=['chromeos-build-alerts@google.com',
+                           'tree', 'build'],
 )
 
 internal_paladin.add_config(constants.BRANCH_UTIL_CONFIG,
@@ -1713,7 +1719,8 @@ internal_paladin.add_config('master-paladin',
   # configuration in the board_config.py code.
   paladin_builder_name='CQ master',
   health_threshold=3,
-  health_alert_recipients=['chromeos-build-alerts@google.com'],
+  health_alert_recipients=['chromeos-build-alerts@google.com',
+                           'tree', 'build'],
   sanity_check_slaves=['link-tot-paladin'],
   trybot_list=False,
 )
diff --git a/cbuildbot/config_dump.json b/cbuildbot/config_dump.json
index c19ee8c65..a1e29dd81 100644
--- a/cbuildbot/config_dump.json
+++ b/cbuildbot/config_dump.json
@@ -38350,7 +38350,9 @@
         "grouped": false,
         "gs_path": "default",
         "health_alert_recipients": [
-            "chromeos-build-alerts@google.com"
+            "chromeos-build-alerts@google.com",
+            "tree",
+            "build"
         ],
         "health_threshold": 3,
         "hw_tests": [],
@@ -61022,7 +61024,9 @@
         "grouped": false,
         "gs_path": "default",
         "health_alert_recipients": [
-            "chromeos-build-alerts@google.com"
+            "chromeos-build-alerts@google.com",
+            "tree",
+            "build"
         ],
         "health_threshold": 1,
         "hw_tests": [],
diff --git a/cbuildbot/constants.py b/cbuildbot/constants.py
index b61d17a61..733784a37 100644
--- a/cbuildbot/constants.py
+++ b/cbuildbot/constants.py
@@ -506,6 +506,12 @@ BUILD_DASHBOARD = 'http://build.chromium.org/p/chromiumos'
 BUILD_INT_DASHBOARD = 'https://uberchromegw.corp.google.com/i/chromeos'
 TRYBOT_DASHBOARD = 'https://uberchromegw.corp.google.com/i/chromiumos.tryserver'
 
+# Valid sherrif types.
+TREE_SHERIFF = 'tree'
+BUILD_DEPUTY = 'build'
+LAB_SHERIFF = 'lab'
+CHROME_GARDENER = 'chrome'
+
 # URLs to retrieve sheriff names from the waterfall.
 TREE_SHERIFF_URL = '%s/sheriff.js' % (BUILD_DASHBOARD)
 TREE_SHERIFF2_URL = '%s/sheriff2.js' % (BUILD_DASHBOARD)
@@ -513,6 +519,14 @@ BUILD_DEPUTY_URL = '%s/chromeos_build_deputy.js' % (BUILD_DASHBOARD)
 LAB_SHERIFF_URL = '%s/sheriff_cros_lab.js' % (BUILD_DASHBOARD)
 CHROME_GARDENER_URL = '%s/sheriff_cr_cros_gardeners.js' % (BUILD_DASHBOARD)
 
+SHERIFF_TYPE_TO_URL = {
+    TREE_SHERIFF: (TREE_SHERIFF_URL, TREE_SHERIFF2_URL),
+    BUILD_DEPUTY: (BUILD_DEPUTY_URL,),
+    LAB_SHERIFF: (LAB_SHERIFF_URL,),
+    CHROME_GARDENER: (CHROME_GARDENER_URL)
+}
+
+
 # Useful config targets.
 CQ_MASTER = 'master-paladin'
 PRE_CQ_GROUP = 'trybot-pre-cq-group'
diff --git a/cbuildbot/stages/completion_stages.py b/cbuildbot/stages/completion_stages.py
index 7e5f9b509..f5ae2dd93 100644
--- a/cbuildbot/stages/completion_stages.py
+++ b/cbuildbot/stages/completion_stages.py
@@ -5,8 +5,6 @@
 """Module containing the completion stages."""
 
 import logging
-import re
-import urllib
 
 from chromite.cbuildbot import commands
 from chromite.cbuildbot import cbuildbot_config
@@ -15,6 +13,7 @@ from chromite.cbuildbot import results_lib
 from chromite.cbuildbot import constants
 from chromite.cbuildbot import manifest_version
 from chromite.cbuildbot import portage_utilities
+from chromite.cbuildbot import tree_status
 from chromite.cbuildbot import validation_pool
 from chromite.cbuildbot.stages import generic_stages
 from chromite.cbuildbot.stages import sync_stages
@@ -394,52 +393,6 @@ class CanaryCompletionStage(MasterSlaveSyncCompletionStage):
     if self._run.config.master:
       self.CanaryMasterHandleFailure(failing, inflight, no_stat)
 
-  @classmethod
-  def _OpenSheriffURL(cls, sheriff_url):
-    """Returns the content of |sheriff_url| or None if failed to open it."""
-    try:
-      response = urllib.urlopen(sheriff_url)
-      if response.getcode() == 200:
-        return response.read()
-    except IOError as e:
-      logging.error('Could not reach %s: %r', sheriff_url, e)
-
-  @classmethod
-  def GetSheriffEmailAddresses(cls, sheriff_type='tree'):
-    """Get the email addresses of the sheriffs or deputy.
-
-    Args:
-      sheriff_type: Type of the sheriff to look for.
-        - 'tree': tree sheriffs
-        - 'build': build deputy
-        - 'lab' : lab sheriff
-        - 'chrome': chrome gardener
-
-    Returns:
-      A list of email addresses.
-    """
-    if sheriff_type == 'tree':
-      urls = [constants.TREE_SHERIFF_URL, constants.TREE_SHERIFF2_URL]
-    elif sheriff_type == 'build':
-      urls = [constants.BUILD_DEPUTY_URL]
-    elif sheriff_type == 'lab':
-      urls = [constants.LAB_SHERIFF_URL]
-    elif sheriff_type == 'chrome':
-      urls = [constants.CHROME_GARDENER_URL]
-    else:
-      raise ValueError('Unknown sheriff type: %s' % sheriff_type)
-
-    sheriffs = []
-    for url in urls:
-      # The URL displays a line: document.write('taco, burrito')
-      raw_line = CanaryCompletionStage._OpenSheriffURL(url)
-      if raw_line is not None:
-        match = re.search(r'\'(.*)\'', raw_line)
-        if match:
-          sheriffs.extend(x.strip() for x in match.group(1).split(','))
-
-    return ['%s%s' % (x, constants.GOOGLE_EMAIL) for x in sheriffs]
-
   def SendCanaryFailureAlert(self, failing, inflight, no_stat):
     """Send an alert email to summarize canary failures.
 
@@ -463,7 +416,7 @@ class CanaryCompletionStage(MasterSlaveSyncCompletionStage):
       # sheriffs. For now, we send it to the build team instead to
       # test the content and make improvements.
       alerts.SendEmail('%s failures' % (builder_name,),
-                       self._run.config.health_alert_recipients,
+                       tree_status.GetHealthAlertRecipients(self._run),
                        message=msg,
                        smtp_server=constants.GOLO_SMTP_SERVER,
                        extra_fields={'X-cbuildbot-alert': 'canary-fail-alert'})
@@ -649,7 +602,7 @@ class CommitQueueCompletionStage(MasterSlaveSyncCompletionStage):
       msg = '\n\n'.join(msgs)
       if not self.ShouldDisableAlerts():
         alerts.SendEmail('%s infra failures' % (builder_name,),
-                         self._run.config.health_alert_recipients,
+                         tree_status.GetHealthAlertRecipients(self._run),
                          message=msg,
                          smtp_server=constants.GOLO_SMTP_SERVER,
                          extra_fields={'X-cbuildbot-alert': 'cq-infra-alert'})
diff --git a/cbuildbot/stages/completion_stages_unittest.py b/cbuildbot/stages/completion_stages_unittest.py
index 2cfd653f1..dd408331a 100755
--- a/cbuildbot/stages/completion_stages_unittest.py
+++ b/cbuildbot/stages/completion_stages_unittest.py
@@ -268,26 +268,6 @@ class CanaryCompletionStageTest(
     return completion_stages.CanaryCompletionStage(
         self._run, sync_stage, success=True)
 
-  def testParsingSheriffEmails(self):
-    """Tests parsing the raw data to get sheriff emails."""
-    # Test parsing when there is only one sheriff.
-    raw_line = "document.write('taco')"
-    self.PatchObject(completion_stages.CanaryCompletionStage,
-                     '_OpenSheriffURL', return_value=raw_line)
-    self.assertEqual(
-        completion_stages.CanaryCompletionStage.GetSheriffEmailAddresses(
-            sheriff_type='build'),
-        ['taco@google.com'])
-
-    # Test parsing when there are multiple sheriffs.
-    raw_line = "document.write('taco, burrito')"
-    self.PatchObject(completion_stages.CanaryCompletionStage,
-                     '_OpenSheriffURL', return_value=raw_line)
-    self.assertEqual(
-        completion_stages.CanaryCompletionStage.GetSheriffEmailAddresses(
-            sheriff_type='build'),
-        ['taco@google.com', 'burrito@google.com'])
-
 
 class CommitQueueCompletionStageTest(
     generic_stages_unittest.AbstractStageTest):
diff --git a/cbuildbot/stages/report_stages.py b/cbuildbot/stages/report_stages.py
index 580355b39..e41d67178 100644
--- a/cbuildbot/stages/report_stages.py
+++ b/cbuildbot/stages/report_stages.py
@@ -9,10 +9,11 @@ import os
 import sys
 
 from chromite.cbuildbot import commands
+from chromite.cbuildbot import constants
 from chromite.cbuildbot import failures_lib
 from chromite.cbuildbot import metadata_lib
 from chromite.cbuildbot import results_lib
-from chromite.cbuildbot import constants
+from chromite.cbuildbot import tree_status
 from chromite.cbuildbot.stages import completion_stages
 from chromite.cbuildbot.stages import generic_stages
 from chromite.cbuildbot.stages import sync_stages
@@ -177,7 +178,7 @@ class ReportStage(generic_stages.BuilderStage,
 
         if not self._run.debug:
           alerts.SendEmail('%s health alert' % builder_run.config.name,
-                           builder_run.config.health_alert_recipients,
+                           tree_status.GetHealthAlertRecipients(builder_run),
                            message=self._HealthAlertMessage(-streak_value),
                            smtp_server=constants.GOLO_SMTP_SERVER,
                            extra_fields={'X-cbuildbot-alert': 'cq-health'})
diff --git a/cbuildbot/stages/report_stages_unittest.py b/cbuildbot/stages/report_stages_unittest.py
index 25ff1dd8e..a91196b9c 100755
--- a/cbuildbot/stages/report_stages_unittest.py
+++ b/cbuildbot/stages/report_stages_unittest.py
@@ -87,7 +87,7 @@ class ReportStageTest(generic_stages_unittest.AbstractStageTest):
   def testAlertEmail(self):
     """Send out alerts when streak counter reaches the threshold."""
     self._Prepare(extra_config={'health_threshold': 3,
-                                'health_alert_recipients': ['fake_recipient']})
+                                'health_alert_recipients': ['foo@bar.org']})
     self._SetupUpdateStreakCounter(counter_value=-3)
     self._SetupCommitQueueSyncPool()
     self.RunStage()
@@ -98,7 +98,7 @@ class ReportStageTest(generic_stages_unittest.AbstractStageTest):
   def testAlertEmailOnFailingStreak(self):
     """Continue sending out alerts when streak counter exceeds the threshold."""
     self._Prepare(extra_config={'health_threshold': 3,
-                                'health_alert_recipients': ['fake_recipient']})
+                                'health_alert_recipients': ['foo@bar.org']})
     self._SetupUpdateStreakCounter(counter_value=-5)
     self._SetupCommitQueueSyncPool()
     self.RunStage()
diff --git a/cbuildbot/tree_status.py b/cbuildbot/tree_status.py
index 147556b10..0a1372a69 100644
--- a/cbuildbot/tree_status.py
+++ b/cbuildbot/tree_status.py
@@ -8,6 +8,7 @@ import httplib
 import json
 import logging
 import os
+import re
 import socket
 import urllib
 import urllib2
@@ -175,3 +176,57 @@ def UpdateTreeStatus(status, message, status_url=None):
       'message': message}
 
   _UpdateTreeStatus(status_url, status_text)
+
+
+def _OpenSheriffURL(sheriff_url):
+  """Returns the content of |sheriff_url| or None if failed to open it."""
+  try:
+    response = urllib.urlopen(sheriff_url)
+    if response.getcode() == 200:
+      return response.read()
+  except IOError as e:
+    logging.error('Could not reach %s: %r', sheriff_url, e)
+
+
+def GetSheriffEmailAddresses(sheriff_type):
+  """Get the email addresses of the sheriffs or deputy.
+
+  Args:
+    sheriff_type: Type of the sheriff to look for. See the keys in
+    constants.SHERIFF_TYPE_TO_URL.
+      - 'tree': tree sheriffs
+      - 'build': build deputy
+      - 'lab' : lab sheriff
+      - 'chrome': chrome gardener
+
+  Returns:
+    A list of email addresses.
+  """
+  if sheriff_type not in constants.SHERIFF_TYPE_TO_URL:
+    raise ValueError('Unknown sheriff type: %s' % sheriff_type)
+
+  urls = constants.SHERIFF_TYPE_TO_URL.get(sheriff_type)
+  sheriffs = []
+  for url in urls:
+    # The URL displays a line: document.write('taco, burrito')
+    raw_line = _OpenSheriffURL(url)
+    if raw_line is not None:
+      match = re.search(r'\'(.*)\'', raw_line)
+      if match:
+        sheriffs.extend(x.strip() for x in match.group(1).split(','))
+
+  return ['%s%s' % (x, constants.GOOGLE_EMAIL) for x in sheriffs]
+
+
+def GetHealthAlertRecipients(builder_run):
+  """Returns a list of email addresses of the health alert recipients."""
+  recipients = []
+  for entry in builder_run.config.health_alert_recipients:
+    if '@' in entry:
+      # If the entry is an email address, add it to the list.
+      recipients.append(entry)
+    else:
+      # Perform address lookup for a non-email entry.
+      recipients.extend(GetSheriffEmailAddresses(entry))
+
+  return recipients
diff --git a/cbuildbot/tree_status_unittest.py b/cbuildbot/tree_status_unittest.py
index 24ca45139..505806f40 100755
--- a/cbuildbot/tree_status_unittest.py
+++ b/cbuildbot/tree_status_unittest.py
@@ -150,5 +150,23 @@ class TestTreeStatus(cros_test_lib.MoxTestCase):
                       period=0.1)
 
 
+class TestGettingSheriffEmails(cros_test_lib.MockTestCase):
+  """Tests functions related to retrieving the sheriff's email address."""
+
+  def testParsingSheriffEmails(self):
+    """Tests parsing the raw data to get sheriff emails."""
+    # Test parsing when there is only one sheriff.
+    raw_line = "document.write('taco')"
+    self.PatchObject(tree_status, '_OpenSheriffURL', return_value=raw_line)
+    self.assertEqual(tree_status.GetSheriffEmailAddresses('build'),
+                     ['taco@google.com'])
+
+    # Test parsing when there are multiple sheriffs.
+    raw_line = "document.write('taco, burrito')"
+    self.PatchObject(tree_status, '_OpenSheriffURL', return_value=raw_line)
+    self.assertEqual(tree_status.GetSheriffEmailAddresses('build'),
+                     ['taco@google.com', 'burrito@google.com'])
+
+
 if __name__ == '__main__':
   cros_test_lib.main()