From ff4512fa00d470990a35939e98ccd421f77997dc Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 10:15:04 +0300 Subject: [PATCH 1/7] more items in comparison test --- tests/test_sync.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 60e5fe5..9941671 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -50,7 +50,7 @@ class TestCalendarSync(unittest.TestCase): def gen_list_to_compare(start, stop): result = [] for i in range(start, stop): - result.append({'iCalUID': 'test{}'.format(i)}) + result.append({'iCalUID': 'test{:06d}'.format(i)}) return result @staticmethod @@ -64,8 +64,11 @@ class TestCalendarSync(unittest.TestCase): return dateutil.parser.parse(start_date) def test_compare(self): - lst_src = TestCalendarSync.gen_list_to_compare(1, 11) - lst_dst = TestCalendarSync.gen_list_to_compare(6, 16) + part_len = 20 + # [1..2n] + lst_src = TestCalendarSync.gen_list_to_compare(1, 1 + part_len * 2) + # [n..3n] + lst_dst = TestCalendarSync.gen_list_to_compare(1 + part_len, 1 + part_len * 3) lst_src_rnd = deepcopy(lst_src) lst_dst_rnd = deepcopy(lst_dst) @@ -76,16 +79,16 @@ class TestCalendarSync(unittest.TestCase): to_ins, to_upd, to_del = CalendarSync._events_list_compare( lst_src_rnd, lst_dst_rnd) - self.assertEqual(len(to_ins), 5) - self.assertEqual(len(to_upd), 5) - self.assertEqual(len(to_del), 5) + self.assertEqual(len(to_ins), part_len) + self.assertEqual(len(to_upd), part_len) + self.assertEqual(len(to_del), part_len) self.assertEqual( - sorted(to_ins, key=lambda x: x['iCalUID']), lst_src[:5]) + sorted(to_ins, key=lambda x: x['iCalUID']), lst_src[:part_len]) self.assertEqual( - sorted(to_del, key=lambda x: x['iCalUID']), lst_dst[5:]) + sorted(to_del, key=lambda x: x['iCalUID']), lst_dst[part_len:]) - to_upd_ok = list(zip(lst_src[5:], lst_dst[:5])) + to_upd_ok = list(zip(lst_src[part_len:], lst_dst[:part_len])) self.assertEqual(len(to_upd), len(to_upd_ok)) for item in to_upd_ok: self.assertIn(item, to_upd) From ebd88322d737f6fc4ebbfd8b99e1dd9b06156f74 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 10:18:08 +0300 Subject: [PATCH 2/7] performance fix in get_item --- gcal_sync/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcal_sync/sync.py b/gcal_sync/sync.py index d7caf3a..e9aafbe 100644 --- a/gcal_sync/sync.py +++ b/gcal_sync/sync.py @@ -40,8 +40,8 @@ class CalendarSync(): keys_to_delete = set(keys_dst) - set(keys_src) def get_item(items, key_val): - items = list(filter(lambda item: item[key] == key_val, items)) - return items[0] + for item in filter(lambda item: item[key] == key_val, items): + return item def items_by_keys(items, key_name, keys): return list(filter(lambda item: item[key_name] in keys, items)) From 2e49e49d0a9970bad796f3ce6497d40a376bde17 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 10:44:55 +0300 Subject: [PATCH 3/7] fix sets in events_list_compare (cherry picked from commit 1cd7baf08c42135e8ee6480a0bf40a73948cedc7) --- gcal_sync/sync.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gcal_sync/sync.py b/gcal_sync/sync.py index e9aafbe..cb7a7ae 100644 --- a/gcal_sync/sync.py +++ b/gcal_sync/sync.py @@ -32,12 +32,12 @@ class CalendarSync(): def get_key(item): return item[key] - keys_src = list(map(get_key, items_src)) - keys_dst = list(map(get_key, items_dst)) + keys_src = set(map(get_key, items_src)) + keys_dst = set(map(get_key, items_dst)) - keys_to_insert = set(keys_src) - set(keys_dst) - keys_to_update = set(keys_src) & set(keys_dst) - keys_to_delete = set(keys_dst) - set(keys_src) + keys_to_insert = keys_src - keys_dst + keys_to_update = keys_src & keys_dst + keys_to_delete = keys_dst - keys_src def get_item(items, key_val): for item in filter(lambda item: item[key] == key_val, items): From 8eb94089a7f657b9d596db8d2f994b4b4a5ee5c2 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 11:36:23 +0300 Subject: [PATCH 4/7] remove old test module --- gcal_sync/test-events.py | 42 ---------------------------------------- 1 file changed, 42 deletions(-) delete mode 100644 gcal_sync/test-events.py diff --git a/gcal_sync/test-events.py b/gcal_sync/test-events.py deleted file mode 100644 index 76c2fcb..0000000 --- a/gcal_sync/test-events.py +++ /dev/null @@ -1,42 +0,0 @@ -import hashlib - -from pytz import UTC, timezone - -import datetime - -def sha1(string): - ''' Хеширование строки - ''' - if isinstance(string, str): - string = string.encode('utf8') - h = hashlib.sha1() - h.update(string) - return h.hexdigest() - -def genenerate(count=10): - ''' Создание тестовых событий - ''' - msk = timezone('Europe/Moscow') - now = UTC.localize(datetime.datetime.utcnow()) - msk_now = msk.normalize(now.astimezone(msk)) - - one_hour = datetime.datetime(1,1,1,2) - datetime.datetime(1,1,1,1) - - start_time = msk_now - (one_hour * 3) - for i in range(count): - event_start = start_time + (one_hour * i) - event_end = event_start + one_hour - updated = UTC.normalize(event_start.astimezone(UTC)).replace(tzinfo=None) - yield { - 'summary': 'test event __ {}'.format(i), - 'location': 'la la la {}'.format(i), - 'description': 'test TEST -- test event {}'.format(i), - 'start': { - 'dateTime': event_start.isoformat() - }, - 'end': { - 'dateTime': event_end.isoformat(), - }, - "iCalUID": "{}@test-domain.ru".format(sha1("test - event {}".format(i))), - "updated": updated.isoformat() + 'Z', - "created": updated.isoformat() + 'Z'} From 8ea8512c116fcd06b9b2f4f064e63999720a662f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 12:36:37 +0300 Subject: [PATCH 5/7] performance fix in compare zip()'ing sorted lists is like 10x faster ( 1.56 seconds instead of 15.05 in test_compare, part_len=5000 ) --- gcal_sync/sync.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/gcal_sync/sync.py b/gcal_sync/sync.py index cb7a7ae..79f62eb 100644 --- a/gcal_sync/sync.py +++ b/gcal_sync/sync.py @@ -39,20 +39,18 @@ class CalendarSync(): keys_to_update = keys_src & keys_dst keys_to_delete = keys_dst - keys_src - def get_item(items, key_val): - for item in filter(lambda item: item[key] == key_val, items): - return item - def items_by_keys(items, key_name, keys): return list(filter(lambda item: item[key_name] in keys, items)) - + items_to_insert = items_by_keys(items_src, key, keys_to_insert) items_to_delete = items_by_keys(items_dst, key, keys_to_delete) - items_to_update = [] - for key_val in keys_to_update: - items_to_update.append( (get_item(items_src, key_val), get_item(items_dst, key_val)) ) - + to_upd_src = items_by_keys(items_src, key, keys_to_update) + to_upd_dst = items_by_keys(items_dst, key, keys_to_update) + to_upd_src.sort(key=get_key) + to_upd_dst.sort(key=get_key) + items_to_update = list(zip(to_upd_src, to_upd_dst)) + return items_to_insert, items_to_update, items_to_delete def _filter_events_to_update(self): From c34b4cdb212d09abc9f5697a5eeeb7d6b3711e7f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 13:49:04 +0300 Subject: [PATCH 6/7] rewrite date/dateTime conversion func's --- gcal_sync/ical.py | 101 +++++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/gcal_sync/ical.py b/gcal_sync/ical.py index 1a71ac1..ec21b8d 100644 --- a/gcal_sync/ical.py +++ b/gcal_sync/ical.py @@ -1,8 +1,47 @@ -from icalendar import Calendar, Event +import datetime import logging + +from icalendar import Calendar, Event from pytz import utc -import datetime + +def format_datetime_utc(value): + """utc datetime as string from date or datetime value + Arguments: + value -- date or datetime value + + Returns: + utc datetime value as string in iso format + """ + if not isinstance(value, datetime.datetime): + value = datetime.datetime( + value.year, value.month, value.day, tzinfo=utc) + value = value.replace(microsecond=1) + return utc.normalize(value.astimezone(utc)).replace(tzinfo=None).isoformat() + 'Z' + + +def gcal_date_or_dateTime(value, check_value=None): + """date or dateTime to gcal (start or end dict) + Arguments: + value -- date or datetime value + check_value - date or datetime to choise result type (if not None) + + Returns: + dict { 'date': ... } or { 'dateTime': ... } + """ + + if check_value is None: + check_value = value + + result = {} + if isinstance(check_value, datetime.datetime): + result['dateTime'] = format_datetime_utc(value) + else: + if isinstance(check_value, datetime.date): + if isinstance(value, datetime.datetime): + value = datetime.date(value.year, value.month, value.day) + result['date'] = value.isoformat() + return result class EventConverter(Event): @@ -32,12 +71,7 @@ class EventConverter(Event): utc datetime value as string in iso format """ - date = self.decoded(prop) - if not isinstance(date, datetime.datetime): - date = datetime.datetime( - date.year, date.month, date.day, tzinfo=utc) - date = date.replace(microsecond=1) - return utc.normalize(date.astimezone(utc)).replace(tzinfo=None).isoformat() + 'Z' + return format_datetime_utc(self.decoded(prop)) def _gcal_start(self): """ event start dict from icalendar event @@ -49,58 +83,31 @@ class EventConverter(Event): dict """ - start_date = self.decoded('DTSTART') - if isinstance(start_date, datetime.datetime): - return { - 'dateTime': self._datetime_str_prop('DTSTART') - } - else: - if isinstance(start_date, datetime.date): - return { - 'date': start_date.isoformat() - } - raise ValueError('DTSTART must be date or datetime') + value = self.decoded('DTSTART') + return gcal_date_or_dateTime(value) def _gcal_end(self): """event end dict from icalendar event Raises: - ValueError -- if DTEND not date or datetime ValueError -- if no DTEND or DURATION - ValueError -- if end date/datetime not found Returns: dict """ + result = None if 'DTEND' in self: - end_date = self.decoded('DTEND') - if isinstance(end_date, datetime.datetime): - return { - 'dateTime': self._datetime_str_prop('DTEND') - } - else: - if isinstance(end_date, datetime.date): - return { - 'date': end_date.isoformat() - } - raise ValueError('DTEND must be date or datetime') - else: - if 'DURATION' in self: - start_date = self.decoded('DTSTART') - duration = self.decoded('DURATION') - end_date = start_date + duration + value = self.decoded('DTEND') + result = gcal_date_or_dateTime(value) + elif 'DURATION' in self: + start_val = self.decoded('DTSTART') + duration = self.decoded('DURATION') + end_val = start_val + duration - if isinstance(start_date, datetime.datetime): - return { - 'dateTime': utc.normalize(end_date.astimezone(utc)).replace(tzinfo=None, microsecond=1).isoformat() + 'Z' - } - else: - if isinstance(start_date, datetime.date): - return { - 'date': datetime.date(end_date.year, end_date.month, end_date.day).isoformat() - } + result = gcal_date_or_dateTime(end_val, check_value=start_val) + else: raise ValueError('no DTEND or DURATION') - raise ValueError('end date/time not found') + return result def _put_to_gcal(self, gcal_event, prop, func, ics_prop=None): """get property from ical event if exist, and put to gcal event From 87a8deafedb3923d48ec0b0d3ce5008d8bce4e53 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Apr 2018 15:41:08 +0300 Subject: [PATCH 7/7] rewrite callback for batch requests --- gcal_sync/gcal.py | 88 +++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/gcal_sync/gcal.py b/gcal_sync/gcal.py index 3e94f0e..fe418e9 100644 --- a/gcal_sync/gcal.py +++ b/gcal_sync/gcal.py @@ -29,6 +29,24 @@ class GoogleCalendarService(): return service +def select_event_key(event): + """select event key for logging + + Arguments: + event -- event resource + + Returns: + key name or None if no key found + """ + + key = None + if 'iCalUID' in event: + key = 'iCalUID' + elif 'id' in event: + key = 'id' + return key + + class GoogleCalendar(): """class to interact with calendar on google """ @@ -39,6 +57,33 @@ class GoogleCalendar(): self.service = service self.calendarId = calendarId + def _make_request_callback(self, action, events_by_req): + """make callback for log result of batch request + + Arguments: + action -- action name + events_by_req -- list of events ordered by request id + + Returns: + callback function + """ + + def callback(request_id, response, exception): + event = events_by_req[int(request_id)] + key = select_event_key(event) + + if exception is not None: + self.logger.error('failed to %s event with %s: %s, exception: %s', + action, key, event.get(key), str(exception)) + else: + resp_key = select_event_key(response) + if resp_key is not None: + event = response + key = resp_key + self.logger.info('event %s ok, %s: %s', + action, key, event.get(key)) + return callback + def list_events_from(self, start): """ list events from calendar, where start date >= start """ @@ -110,15 +155,7 @@ class GoogleCalendar(): fields = 'id' events_by_req = [] - def insert_callback(request_id, response, exception): - if exception is not None: - event = events_by_req[int(request_id)] - self.logger.error('failed to insert event with UID: %s, exception: %s', event.get( - 'UID'), str(exception)) - else: - event = response - self.logger.info('event created, id: %s', event.get('id')) - + insert_callback = self._make_request_callback('insert', events_by_req) batch = self.service.new_batch_http_request(callback=insert_callback) i = 0 for event in events: @@ -138,15 +175,7 @@ class GoogleCalendar(): fields = 'id' events_by_req = [] - def patch_callback(request_id, response, exception): - if exception is not None: - event = events_by_req[int(request_id)] - self.logger.error('failed to patch event with UID: %s, exception: %s', event.get( - 'UID'), str(exception)) - else: - event = response - self.logger.info('event patched, id: %s', event.get('id')) - + patch_callback = self._make_request_callback('patch', events_by_req) batch = self.service.new_batch_http_request(callback=patch_callback) i = 0 for event_new, event_old in event_tuples: @@ -168,15 +197,7 @@ class GoogleCalendar(): fields = 'id' events_by_req = [] - def update_callback(request_id, response, exception): - if exception is not None: - event = events_by_req[int(request_id)] - self.logger.error('failed to update event with UID: %s, exception: %s', event.get( - 'UID'), str(exception)) - else: - event = response - self.logger.info('event updated, id: %s', event.get('id')) - + update_callback = self._make_request_callback('update', events_by_req) batch = self.service.new_batch_http_request(callback=update_callback) i = 0 for event_new, event_old in event_tuples: @@ -197,14 +218,7 @@ class GoogleCalendar(): events_by_req = [] - def delete_callback(request_id, _, exception): - event = events_by_req[int(request_id)] - if exception is not None: - self.logger.error('failed to delete event with UID: %s, exception: %s', event.get( - 'UID'), str(exception)) - else: - self.logger.info('event deleted, id: %s', event.get('id')) - + delete_callback = self._make_request_callback('delete', events_by_req) batch = self.service.new_batch_http_request(callback=delete_callback) i = 0 for event in events: @@ -216,10 +230,10 @@ class GoogleCalendar(): def create(self, summary, timeZone=None): """create calendar - + Arguments: summary -- new calendar summary - + Keyword Arguments: timeZone -- new calendar timezone as string (optional)