diff options
author | Jongmok Hong <jongmok@google.com> | 2018-11-01 15:18:24 +0900 |
---|---|---|
committer | Jongmok Hong <jongmok@google.com> | 2018-11-02 10:17:28 +0900 |
commit | 331fa9ff692c513fbda43a62ad6ada878114463a (patch) | |
tree | 345694da6d893375c6c7cadbbc46d9d55c4fa3d1 | |
parent | 988ae09a5613cf8eb3cb75f31e450b243d5e1d0e (diff) | |
download | test_serving-331fa9ff692c513fbda43a62ad6ada878114463a.tar.gz |
Remove outdated devices.
Test: python testing/e2e_test.py, go/vtslab-schedule-dev
Bug: 117583754
Change-Id: I2139e796805287d2e85b8d3fb7819af9a5ec6dfc
-rw-r--r-- | gae/cron.yaml | 3 | ||||
-rw-r--r-- | gae/webapp/src/tasks/removing_outdated_devices.py | 37 | ||||
-rw-r--r-- | gae/webapp/src/tasks/removing_outdated_devices_test.py | 111 | ||||
-rw-r--r-- | gae/webapp/src/webapp_main.py | 3 |
4 files changed, 154 insertions, 0 deletions
diff --git a/gae/cron.yaml b/gae/cron.yaml index 20e23cd..e4d4a71 100644 --- a/gae/cron.yaml +++ b/gae/cron.yaml @@ -8,3 +8,6 @@ cron: - description: "job_heartbeat" url: /tasks/job_heartbeat schedule: every 1 minutes +- description: "remove_outdated_devices" + url: /tasks/remove_outdated_devices + schedule: every 1 hours diff --git a/gae/webapp/src/tasks/removing_outdated_devices.py b/gae/webapp/src/tasks/removing_outdated_devices.py new file mode 100644 index 0000000..6a4eeab --- /dev/null +++ b/gae/webapp/src/tasks/removing_outdated_devices.py @@ -0,0 +1,37 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import webapp2 + +from google.appengine.ext import ndb + +from webapp.src.proto import model + +OUTDATED_DEVICE_REMOVE_TIME_IN_HOURS = 48 + + +class RemoveOutdatedDevices(webapp2.RequestHandler): + """Main class for /tasks/remove_outdated_devices. + + Used to find outdated devices and remove them. + """ + + def get(self): + device_query = model.DeviceModel.query( + model.DeviceModel.timestamp < datetime.datetime.now() - + datetime.timedelta(hours=OUTDATED_DEVICE_REMOVE_TIME_IN_HOURS)) + outdated_devices = device_query.fetch(keys_only=True) + ndb.delete_multi(outdated_devices) diff --git a/gae/webapp/src/tasks/removing_outdated_devices_test.py b/gae/webapp/src/tasks/removing_outdated_devices_test.py new file mode 100644 index 0000000..6ff57d4 --- /dev/null +++ b/gae/webapp/src/tasks/removing_outdated_devices_test.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import unittest + +try: + from unittest import mock +except ImportError: + import mock + +from webapp.src.proto import model +from webapp.src.tasks import removing_outdated_devices +from webapp.src.testing import unittest_base + + +class RemoveOutdatedDevicesTest(unittest_base.UnitTestBase): + """Tests for RemoveOutdatedDevices cron class. + + Attributes: + remove_outdated_device: A mock device_heartbeat.RemoveOutdatedDevices + instance. + """ + + def setUp(self): + """Initializes test""" + super(RemoveOutdatedDevicesTest, self).setUp() + # Mocking RemoveOutdatedDevices and essential methods. + self.remove_outdated_device = ( + removing_outdated_devices.RemoveOutdatedDevices(mock.Mock())) + self.remove_outdated_device.response = mock.Mock() + self.remove_outdated_device.response.write = mock.Mock() + + def testRemoveOutdatedDevicesTest(self): + """Asserts job heartbeat detects unavailable jobs.""" + device_a_serial = "a" + device_b_serial = "b" + device_c_serial = "c" + device_d_serial = "c" + + # create a device A, which is outdated. + device = self.GenerateDeviceModel(serial=device_a_serial) + device.timestamp = datetime.datetime.now() - datetime.timedelta( + hours=100) + device.put() + + # create a device B, which is offline for a day. + device = self.GenerateDeviceModel(serial=device_b_serial) + device.timestamp = datetime.datetime.now() - datetime.timedelta( + hours=24) + device.put() + + # create a device C and D, which are alive. + for serial in [device_c_serial, device_d_serial]: + device = self.GenerateDeviceModel(serial=serial) + device.timestamp = datetime.datetime.now() + device.put() + + # Remove outdated devices. + self.remove_outdated_device.get() + + devices = model.DeviceModel.query().fetch() + + # device A should not be included. + self.assertEqual(len(devices), 3) + self.assertTrue(device_a_serial not in [x.serial for x in devices]) + + # change devices' timestamp + for device in devices: + device.timestamp = device.timestamp - datetime.timedelta(hours=25) + device.put() + + # Remove outdated devices. + self.remove_outdated_device.get() + + devices = model.DeviceModel.query().fetch() + + # Now device B should not be included also. + self.assertEqual(len(devices), 2) + self.assertTrue(device_b_serial not in [x.serial for x in devices]) + + # change devices' timestamp + for device in devices: + device.timestamp = device.timestamp - datetime.timedelta(hours=25) + device.put() + + # Remove outdated devices. + self.remove_outdated_device.get() + + devices = model.DeviceModel.query().fetch() + + # Now there should not be no devices. + self.assertEqual(len(devices), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/gae/webapp/src/webapp_main.py b/gae/webapp/src/webapp_main.py index eaaa5ee..2587f4b 100644 --- a/gae/webapp/src/webapp_main.py +++ b/gae/webapp/src/webapp_main.py @@ -24,6 +24,7 @@ from webapp.src.scheduler import device_heartbeat from webapp.src.scheduler import job_heartbeat from webapp.src.scheduler import periodic from webapp.src.tasks import indexing +from webapp.src.tasks import removing_outdated_devices class RedirectHandler(base.BaseHandler): @@ -55,6 +56,8 @@ app = webapp2.WSGIApplication( ("/tasks/schedule", periodic.PeriodicScheduler), ("/tasks/device_heartbeat", device_heartbeat.PeriodicDeviceHeartBeat), ("/tasks/job_heartbeat", job_heartbeat.PeriodicJobHeartBeat), + ("/tasks/remove_outdated_devices", + removing_outdated_devices.RemoveOutdatedDevices), ("/tasks/indexing([/]?.*)", indexing.CreateIndex), ("/redirect/(.*)", RedirectHandler), ], |