Merge branch 'develop'

* develop:
  rewrite callback for batch requests
  rewrite date/dateTime conversion func's
  performance fix in compare
  remove old test module
  fix sets in events_list_compare
  performance fix in get_item
  more items in comparison test

gcal_sync/gcal.py

@@ -29,6 +29,24 @@ class GoogleCalendarService():
         return service
 
 
+def select_event_key(event):
+    """select event key for logging
+    
+    Arguments:
+        event -- event resource
+    
+    Returns:
+        key name or None if no key found
+    """
+
+    key = None
+    if 'iCalUID' in event:
+        key = 'iCalUID'
+    elif 'id' in event:
+        key = 'id'
+    return key
+
+
 class GoogleCalendar():
     """class to interact with calendar on google
     """
@@ -39,6 +57,33 @@ class GoogleCalendar():
         self.service = service
         self.calendarId = calendarId
 
+    def _make_request_callback(self, action, events_by_req):
+        """make callback for log result of batch request
+        
+        Arguments:
+            action -- action name
+            events_by_req -- list of events ordered by request id
+        
+        Returns:
+            callback function
+        """
+
+        def callback(request_id, response, exception):
+            event = events_by_req[int(request_id)]
+            key = select_event_key(event)
+
+            if exception is not None:
+                self.logger.error('failed to %s event with %s: %s, exception: %s',
+                                  action, key, event.get(key), str(exception))
+            else:
+                resp_key = select_event_key(response)
+                if resp_key is not None:
+                    event = response
+                    key = resp_key
+                self.logger.info('event %s ok, %s: %s',
+                                 action, key, event.get(key))
+        return callback
+
     def list_events_from(self, start):
         """ list events from calendar, where start date >= start
         """
@@ -110,15 +155,7 @@ class GoogleCalendar():
         fields = 'id'
         events_by_req = []
 
-        def insert_callback(request_id, response, exception):
-            if exception is not None:
-                event = events_by_req[int(request_id)]
-                self.logger.error('failed to insert event with UID: %s, exception: %s', event.get(
-                    'UID'), str(exception))
-            else:
-                event = response
-                self.logger.info('event created, id: %s', event.get('id'))
-
+        insert_callback = self._make_request_callback('insert', events_by_req)
         batch = self.service.new_batch_http_request(callback=insert_callback)
         i = 0
         for event in events:
@@ -138,15 +175,7 @@ class GoogleCalendar():
         fields = 'id'
         events_by_req = []
 
-        def patch_callback(request_id, response, exception):
-            if exception is not None:
-                event = events_by_req[int(request_id)]
-                self.logger.error('failed to patch event with UID: %s, exception: %s', event.get(
-                    'UID'), str(exception))
-            else:
-                event = response
-                self.logger.info('event patched, id: %s', event.get('id'))
-
+        patch_callback = self._make_request_callback('patch', events_by_req)
         batch = self.service.new_batch_http_request(callback=patch_callback)
         i = 0
         for event_new, event_old in event_tuples:
@@ -168,15 +197,7 @@ class GoogleCalendar():
         fields = 'id'
         events_by_req = []
 
-        def update_callback(request_id, response, exception):
-            if exception is not None:
-                event = events_by_req[int(request_id)]
-                self.logger.error('failed to update event with UID: %s, exception: %s', event.get(
-                    'UID'), str(exception))
-            else:
-                event = response
-                self.logger.info('event updated, id: %s', event.get('id'))
-
+        update_callback = self._make_request_callback('update', events_by_req)
         batch = self.service.new_batch_http_request(callback=update_callback)
         i = 0
         for event_new, event_old in event_tuples:
@@ -197,14 +218,7 @@ class GoogleCalendar():
 
         events_by_req = []
 
-        def delete_callback(request_id, _, exception):
-            event = events_by_req[int(request_id)]
-            if exception is not None:
-                self.logger.error('failed to delete event with UID: %s, exception: %s', event.get(
-                    'UID'), str(exception))
-            else:
-                self.logger.info('event deleted, id: %s', event.get('id'))
-
+        delete_callback = self._make_request_callback('delete', events_by_req)
         batch = self.service.new_batch_http_request(callback=delete_callback)
         i = 0
         for event in events:
@@ -216,10 +230,10 @@ class GoogleCalendar():
 
     def create(self, summary, timeZone=None):
         """create calendar
-        
+
         Arguments:
             summary -- new calendar summary
-        
+
         Keyword Arguments:
             timeZone -- new calendar timezone as string (optional)
 

gcal_sync/ical.py

@@ -1,8 +1,47 @@
-from icalendar import Calendar, Event
+import datetime
 import logging
+
+from icalendar import Calendar, Event
 from pytz import utc
 
-import datetime
+
+def format_datetime_utc(value):
+    """utc datetime as string from date or datetime value
+    Arguments:
+        value -- date or datetime value
+
+    Returns:
+        utc datetime value as string in iso format
+    """
+    if not isinstance(value, datetime.datetime):
+        value = datetime.datetime(
+            value.year, value.month, value.day, tzinfo=utc)
+    value = value.replace(microsecond=1)
+    return utc.normalize(value.astimezone(utc)).replace(tzinfo=None).isoformat() + 'Z'
+
+
+def gcal_date_or_dateTime(value, check_value=None):
+    """date or dateTime to gcal (start or end dict)
+    Arguments:
+        value -- date or datetime value
+        check_value - date or datetime to choise result type (if not None)
+
+    Returns:
+        dict { 'date': ... } or { 'dateTime': ... }
+    """
+
+    if check_value is None:
+        check_value = value
+
+    result = {}
+    if isinstance(check_value, datetime.datetime):
+        result['dateTime'] = format_datetime_utc(value)
+    else:
+        if isinstance(check_value, datetime.date):
+            if isinstance(value, datetime.datetime):
+                value = datetime.date(value.year, value.month, value.day)
+        result['date'] = value.isoformat()
+    return result
 
 
 class EventConverter(Event):
@@ -32,12 +71,7 @@ class EventConverter(Event):
             utc datetime value as string in iso format
         """
 
-        date = self.decoded(prop)
-        if not isinstance(date, datetime.datetime):
-            date = datetime.datetime(
-                date.year, date.month, date.day, tzinfo=utc)
-        date = date.replace(microsecond=1)
-        return utc.normalize(date.astimezone(utc)).replace(tzinfo=None).isoformat() + 'Z'
+        return format_datetime_utc(self.decoded(prop))
 
     def _gcal_start(self):
         """ event start dict from icalendar event
@@ -49,58 +83,31 @@ class EventConverter(Event):
             dict
         """
 
-        start_date = self.decoded('DTSTART')
-        if isinstance(start_date, datetime.datetime):
-            return {
-                'dateTime': self._datetime_str_prop('DTSTART')
-            }
-        else:
-            if isinstance(start_date, datetime.date):
-                return {
-                    'date': start_date.isoformat()
-                }
-            raise ValueError('DTSTART must be date or datetime')
+        value = self.decoded('DTSTART')
+        return gcal_date_or_dateTime(value)
 
     def _gcal_end(self):
         """event end dict from icalendar event
 
         Raises:
-            ValueError -- if DTEND not date or datetime
             ValueError -- if no DTEND or DURATION
-            ValueError -- if end date/datetime not found
         Returns:
             dict
         """
 
+        result = None
         if 'DTEND' in self:
-            end_date = self.decoded('DTEND')
-            if isinstance(end_date, datetime.datetime):
-                return {
-                    'dateTime': self._datetime_str_prop('DTEND')
-                }
-            else:
-                if isinstance(end_date, datetime.date):
-                    return {
-                        'date': end_date.isoformat()
-                    }
-                raise ValueError('DTEND must be date or datetime')
-        else:
-            if 'DURATION' in self:
-                start_date = self.decoded('DTSTART')
-                duration = self.decoded('DURATION')
-                end_date = start_date + duration
+            value = self.decoded('DTEND')
+            result = gcal_date_or_dateTime(value)
+        elif 'DURATION' in self:
+            start_val = self.decoded('DTSTART')
+            duration = self.decoded('DURATION')
+            end_val = start_val + duration
 
-                if isinstance(start_date, datetime.datetime):
-                    return {
-                        'dateTime': utc.normalize(end_date.astimezone(utc)).replace(tzinfo=None, microsecond=1).isoformat() + 'Z'
-                    }
-                else:
-                    if isinstance(start_date, datetime.date):
-                        return {
-                            'date': datetime.date(end_date.year, end_date.month, end_date.day).isoformat()
-                        }
+            result = gcal_date_or_dateTime(end_val, check_value=start_val)
+        else:
             raise ValueError('no DTEND or DURATION')
-        raise ValueError('end date/time not found')
+        return result
 
     def _put_to_gcal(self, gcal_event, prop, func, ics_prop=None):
         """get property from ical event if exist, and put to gcal event

gcal_sync/sync.py

@@ -32,27 +32,25 @@ class CalendarSync():
 
         def get_key(item): return item[key]
 
-        keys_src = list(map(get_key, items_src))
-        keys_dst = list(map(get_key, items_dst))
+        keys_src = set(map(get_key, items_src))
+        keys_dst = set(map(get_key, items_dst))
 
-        keys_to_insert = set(keys_src) - set(keys_dst)
-        keys_to_update = set(keys_src) & set(keys_dst)
-        keys_to_delete = set(keys_dst) - set(keys_src)
-
-        def get_item(items, key_val):
-            items = list(filter(lambda item: item[key] == key_val, items))
-            return items[0]
+        keys_to_insert = keys_src - keys_dst
+        keys_to_update = keys_src & keys_dst
+        keys_to_delete = keys_dst - keys_src
 
         def items_by_keys(items, key_name, keys):
             return list(filter(lambda item: item[key_name] in keys, items))
-        
+
         items_to_insert = items_by_keys(items_src, key, keys_to_insert)
         items_to_delete = items_by_keys(items_dst, key, keys_to_delete)
 
-        items_to_update = []
-        for key_val in keys_to_update:
-            items_to_update.append( (get_item(items_src, key_val), get_item(items_dst, key_val)) )
-        
+        to_upd_src = items_by_keys(items_src, key, keys_to_update)
+        to_upd_dst = items_by_keys(items_dst, key, keys_to_update)
+        to_upd_src.sort(key=get_key)
+        to_upd_dst.sort(key=get_key)
+        items_to_update = list(zip(to_upd_src, to_upd_dst))
+
         return items_to_insert, items_to_update, items_to_delete
 
     def _filter_events_to_update(self):

gcal_sync/test-events.py

@@ -1,42 +0,0 @@
-import hashlib
-
-from pytz import UTC, timezone
-
-import datetime
-
-def sha1(string):
-    ''' Хеширование строки
-    '''
-    if isinstance(string, str):
-        string = string.encode('utf8')
-    h = hashlib.sha1()
-    h.update(string)
-    return h.hexdigest()
-
-def genenerate(count=10):
-    ''' Создание тестовых событий
-    '''
-    msk = timezone('Europe/Moscow')
-    now = UTC.localize(datetime.datetime.utcnow())
-    msk_now = msk.normalize(now.astimezone(msk))
-
-    one_hour = datetime.datetime(1,1,1,2) - datetime.datetime(1,1,1,1)
-    
-    start_time = msk_now - (one_hour * 3)
-    for i in range(count):
-        event_start = start_time + (one_hour * i)
-        event_end   = event_start + one_hour
-        updated = UTC.normalize(event_start.astimezone(UTC)).replace(tzinfo=None)
-        yield {
-            'summary': 'test event __ {}'.format(i),
-            'location': 'la la la {}'.format(i),
-            'description': 'test TEST -- test event {}'.format(i),
-            'start': {
-                'dateTime': event_start.isoformat()
-            },
-            'end': {
-                'dateTime': event_end.isoformat(),
-            },
-            "iCalUID": "{}@test-domain.ru".format(sha1("test - event {}".format(i))),
-            "updated": updated.isoformat() + 'Z',
-            "created": updated.isoformat() + 'Z'}

tests/test_sync.py

@@ -50,7 +50,7 @@ class TestCalendarSync(unittest.TestCase):
     def gen_list_to_compare(start, stop):
         result = []
         for i in range(start, stop):
-            result.append({'iCalUID': 'test{}'.format(i)})
+            result.append({'iCalUID': 'test{:06d}'.format(i)})
         return result
 
     @staticmethod
@@ -64,8 +64,11 @@ class TestCalendarSync(unittest.TestCase):
         return dateutil.parser.parse(start_date)
 
     def test_compare(self):
-        lst_src = TestCalendarSync.gen_list_to_compare(1, 11)
-        lst_dst = TestCalendarSync.gen_list_to_compare(6, 16)
+        part_len = 20
+        # [1..2n]
+        lst_src = TestCalendarSync.gen_list_to_compare(1, 1 + part_len * 2)
+        # [n..3n]
+        lst_dst = TestCalendarSync.gen_list_to_compare(1 + part_len, 1 + part_len * 3)
 
         lst_src_rnd = deepcopy(lst_src)
         lst_dst_rnd = deepcopy(lst_dst)
@@ -76,16 +79,16 @@ class TestCalendarSync(unittest.TestCase):
         to_ins, to_upd, to_del = CalendarSync._events_list_compare(
             lst_src_rnd, lst_dst_rnd)
 
-        self.assertEqual(len(to_ins), 5)
-        self.assertEqual(len(to_upd), 5)
-        self.assertEqual(len(to_del), 5)
+        self.assertEqual(len(to_ins), part_len)
+        self.assertEqual(len(to_upd), part_len)
+        self.assertEqual(len(to_del), part_len)
 
         self.assertEqual(
-            sorted(to_ins, key=lambda x: x['iCalUID']), lst_src[:5])
+            sorted(to_ins, key=lambda x: x['iCalUID']), lst_src[:part_len])
         self.assertEqual(
-            sorted(to_del, key=lambda x: x['iCalUID']), lst_dst[5:])
+            sorted(to_del, key=lambda x: x['iCalUID']), lst_dst[part_len:])
 
-        to_upd_ok = list(zip(lst_src[5:], lst_dst[:5]))
+        to_upd_ok = list(zip(lst_src[part_len:], lst_dst[:part_len]))
         self.assertEqual(len(to_upd), len(to_upd_ok))
         for item in to_upd_ok:
             self.assertIn(item, to_upd)