From 030ad3e836009522e2fa189b2b96f340dd2e6346 Mon Sep 17 00:00:00 2001 From: Rachel Fenichel Date: Tue, 18 Aug 2020 15:22:33 -0700 Subject: [PATCH 1/3] Add expiration handler --- appengine/expiration.py | 49 +++++++++++++++++++++++++++++++++++++++++ appengine/main.py | 3 +++ 2 files changed, 52 insertions(+) create mode 100644 appengine/expiration.py diff --git a/appengine/expiration.py b/appengine/expiration.py new file mode 100644 index 000000000..d095f2756 --- /dev/null +++ b/appengine/expiration.py @@ -0,0 +1,49 @@ +""" +Copyright 2020 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""Delete expired XML. +""" + +__author__ = "fenichel@google.com (Rachel Fenichel)" + + +import storage +import datetime + +from google.cloud import ndb + +def delete_expired(): + # Delete entries that have not been accessed in more than a year. + EXPIRATION_DAYS = 365 + # Limit the query to avoid timeouts. + QUERY_LIMIT = 1000 + bestBefore = datetime.datetime.utcnow() - datetime.timedelta(days=EXPIRATION_DAYS) + client = ndb.Client() + with client.context(): + query = storage.Xml.query(storage.Xml.last_accessed < bestBefore) + results = query.fetch(limit=QUERY_LIMIT, keys_only=True) + for x in results: + x.delete() + + +def app(environ, start_response): + out = "" + headers = [ + ("Content-Type", "text/plain") + ] + start_response("200 OK", headers) + delete_expired() + return [out.encode("utf-8")] diff --git a/appengine/main.py b/appengine/main.py index 349c4f0c1..2b0918cf7 100644 --- a/appengine/main.py +++ b/appengine/main.py @@ -15,6 +15,7 @@ limitations under the License. """ import storage +import expiration # Route to requested handler. @@ -23,6 +24,8 @@ def app(environ, start_response): return redirect(environ, start_response) if environ["PATH_INFO"] == "/storage": return storage.app(environ, start_response) + if environ["PATH_INFO"] == "/expiration": + return retention_job.app(environ, start_response) start_response("404 Not Found", []) return [b"Page not found."] From 441a1351088877fedf6dd6c0bd8ff7f37975f845 Mon Sep 17 00:00:00 2001 From: Rachel Fenichel Date: Tue, 18 Aug 2020 15:24:33 -0700 Subject: [PATCH 2/3] Fix naming" --- appengine/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/main.py b/appengine/main.py index 2b0918cf7..765087c61 100644 --- a/appengine/main.py +++ b/appengine/main.py @@ -25,7 +25,7 @@ def app(environ, start_response): if environ["PATH_INFO"] == "/storage": return storage.app(environ, start_response) if environ["PATH_INFO"] == "/expiration": - return retention_job.app(environ, start_response) + return expiration.app(environ, start_response) start_response("404 Not Found", []) return [b"Page not found."] From b1442513a297331db3dffe94c962a543f74f1fae Mon Sep 17 00:00:00 2001 From: Rachel Fenichel Date: Mon, 24 Aug 2020 12:52:03 -0700 Subject: [PATCH 3/3] Review comments --- appengine/expiration.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/appengine/expiration.py b/appengine/expiration.py index d095f2756..19142c30a 100644 --- a/appengine/expiration.py +++ b/appengine/expiration.py @@ -25,11 +25,13 @@ import datetime from google.cloud import ndb + +EXPIRATION_DAYS = 365 +# Limit the query to avoid timeouts. +QUERY_LIMIT = 1000 + def delete_expired(): - # Delete entries that have not been accessed in more than a year. - EXPIRATION_DAYS = 365 - # Limit the query to avoid timeouts. - QUERY_LIMIT = 1000 + """Deletes entries that have not been accessed in more than a year.""" bestBefore = datetime.datetime.utcnow() - datetime.timedelta(days=EXPIRATION_DAYS) client = ndb.Client() with client.context():