Initial commit

.gitignore

@@ -0,0 +1,5 @@
+config.yml
+service-account.json
+*.pyc
+my-test*.ics
+.vscode/*

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Dmitry Belyaev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+# sync_ics2gcal
+
+Python script for sync .ics file with Google calendar

gcal_sync/__init__.py

@@ -0,0 +1,14 @@
+
+from .ical import (
+    CalendarConverter,
+    EventConverter
+)
+
+from .gcal import (
+    GoogleCalendarService,
+    GoogleCalendar
+)
+
+from .sync import (
+    CalendarSync
+)

gcal_sync/gcal.py

@@ -0,0 +1,215 @@
+from apiclient import discovery
+import httplib2
+import logging
+from oauth2client import service_account
+from pytz import utc
+import sys
+
+
+class GoogleCalendarService():
+    @staticmethod
+    def from_srv_acc_file(service_account_file):
+        scopes = 'https://www.googleapis.com/auth/calendar'
+        credentials = service_account.ServiceAccountCredentials.from_json_keyfile_name(
+            service_account_file, scopes=scopes)
+        http = credentials.authorize(httplib2.Http())
+        service = discovery.build('calendar', 'v3', http=http)
+        return service
+
+
+class GoogleCalendar():
+    logger = logging.getLogger('GoogleCalendar')
+
+    def __init__(self, service, calendarId):
+        self.service = service
+        self.calendarId = calendarId
+
+    def list_events_from(self, start):
+        ''' Получение списка событий из GCAL начиная с даты start
+        '''
+        events = []
+        page_token = None
+        timeMin = utc.normalize(start.astimezone(utc)).replace(
+            tzinfo=None).isoformat() + 'Z'
+        while True:
+            response = self.service.events().list(calendarId=self.calendarId, pageToken=page_token,
+                                                  singleEvents=True, timeMin=timeMin, fields='id,iCalUID,updated').execute()
+            if 'items' in response:
+                events.extend(response['items'])
+                page_token = response.get('nextPageToken')
+                if not page_token:
+                    break
+        self.logger.info('%d events listed', len(events))
+        return events
+
+    def find_exists(self, events):
+        """ Поиск уже существующих в GCAL событий, из списка событий к вставке
+
+        Arguments:
+            events {list} -- list of events
+
+        Returns:
+            tuple -- (events_exist, events_not_found)
+                  events_exist - list of tuples: (new_event, exists_event)
+        """
+
+        events_by_req = []
+        exists = []
+        not_found = []
+
+        def list_callback(request_id, response, exception):
+            found = False
+            event = events_by_req[int(request_id)]
+            if exception is None:
+                found = ([] != response['items'])
+            else:
+                self.logger.error('exception %s, while listing event with UID: %s', str(
+                    exception), event['iCalUID'])
+            if found:
+                exists.append(
+                    (event, response['items'][0]))
+            else:
+                not_found.append(events_by_req[int(request_id)])
+
+        batch = self.service.new_batch_http_request(callback=list_callback)
+        i = 0
+        for event in events:
+            events_by_req.append(event)
+            batch.add(self.service.events().list(calendarId=self.calendarId,
+                                                 iCalUID=event['iCalUID'], showDeleted=True, fields='id,iCalUID,updated'), request_id=str(i))
+            i += 1
+        batch.execute()
+        self.logger.info('%d events exists, %d not found',
+                         len(exists), len(not_found))
+        return exists, not_found
+
+    def insert_events(self, events):
+        """ Вставка событий в GCAL
+
+        Arguments:
+            events  -- список событий
+        """
+
+        events_by_req = []
+
+        def insert_callback(request_id, response, exception):
+            if exception is not None:
+                event = events_by_req[int(request_id)]
+                self.logger.error('failed to insert event with UID: %s, exception: %s', event.get(
+                    'UID'), str(exception))
+            else:
+                event = response
+                self.logger.info('event created, id: %s', event.get('id'))
+
+        batch = self.service.new_batch_http_request(callback=insert_callback)
+        i = 0
+        for event in events:
+            events_by_req.append(event)
+            batch.add(self.service.events().insert(
+                calendarId=self.calendarId, body=event, fields='id,iCalUID,updated'), request_id=str(i))
+            i += 1
+        batch.execute()
+
+    def patch_events(self, event_tuples):
+        """ Обновление (патч) событий в GCAL
+
+        Arguments:
+            calendarId  -- ИД календаря
+            event_tuples  -- список кортежей событий (новое, старое)
+        """
+
+        events_by_req = []
+
+        def patch_callback(request_id, response, exception):
+            if exception is not None:
+                event = events_by_req[int(request_id)]
+                self.logger.error('failed to patch event with UID: %s, exception: %s', event.get(
+                    'UID'), str(exception))
+            else:
+                event = response
+                self.logger.info('event patched, id: %s', event.get('id'))
+
+        batch = self.service.new_batch_http_request(callback=patch_callback)
+        i = 0
+        for event_new, event_old in event_tuples:
+            if 'id' not in event_old:
+                continue
+            events_by_req.append(event_new)
+            batch.add(self.service.events().patch(
+                calendarId=self.calendarId, eventId=event_old['id'], body=event_new), fields='id,iCalUID,updated', request_id=str(i))
+            i += 1
+        batch.execute()
+
+    def update_events(self, event_tuples):
+        """ Обновление событий в GCAL
+
+        Arguments:
+            event_tuples  -- список кортежей событий (новое, старое)
+        """
+
+        events_by_req = []
+
+        def update_callback(request_id, response, exception):
+            if exception is not None:
+                event = events_by_req[int(request_id)]
+                self.logger.error('failed to update event with UID: %s, exception: %s', event.get(
+                    'UID'), str(exception))
+            else:
+                event = response
+                self.logger.info('event updated, id: %s', event.get('id'))
+
+        batch = self.service.new_batch_http_request(callback=update_callback)
+        i = 0
+        for event_new, event_old in event_tuples:
+            if 'id' not in event_old:
+                continue
+            events_by_req.append(event_new)
+            batch.add(self.service.events().update(
+                calendarId=self.calendarId, eventId=event_old['id'], body=event_new, fields='id,iCalUID,updated'), request_id=str(i))
+            i += 1
+        batch.execute()
+
+    def delete_events(self, events):
+        """ Удаление событий в GCAL
+
+        Arguments:
+            events  -- список событий
+        """
+
+        events_by_req = []
+
+        def delete_callback(request_id, _, exception):
+            event = events_by_req[int(request_id)]
+            if exception is not None:
+                self.logger.error('failed to delete event with UID: %s, exception: %s', event.get(
+                    'UID'), str(exception))
+            else:
+                self.logger.info('event deleted, id: %s', event.get('id'))
+
+        batch = self.service.new_batch_http_request(callback=delete_callback)
+        i = 0
+        for event in events:
+            events_by_req.append(event)
+            batch.add(self.service.events().delete(
+                calendarId=self.calendarId, eventId=event['id']), request_id=str(i))
+            i += 1
+        batch.execute()
+
+    def make_public(self):
+        rule_public = {
+            'scope': {
+                'type': 'default',
+            },
+            'role': 'reader'
+        }
+        return self.service.acl().insert(calendarId=self.calendarId, body=rule_public).execute()
+
+    def add_owner(self, email):
+        rule_owner = {
+            'scope': {
+                'type': 'user',
+                'value': email,
+            },
+            'role': 'owner'
+        }
+        return self.service.acl().insert(calendarId=self.calendarId, body=rule_owner).execute()

gcal_sync/ical.py

@@ -0,0 +1,126 @@
+from icalendar import Calendar, Event
+import logging
+from pytz import utc
+
+import datetime
+
+
+class EventConverter(Event):
+    """Convert icalendar event to google calendar resource
+    ( https://developers.google.com/calendar/v3/reference/events#resource-representations )
+    """
+
+    def _str_prop(self, prop):
+        return self.decoded(prop).decode(encoding='utf-8')
+
+    def _datetime_str_prop(self, prop):
+        date = self.decoded(prop)
+        if not isinstance(date, datetime.datetime):
+            date = datetime.datetime(
+                date.year, date.month, date.day, tzinfo=utc)
+        date = date.replace(microsecond=1)
+        return utc.normalize(date.astimezone(utc)).replace(tzinfo=None).isoformat() + 'Z'
+
+    def _gcal_start(self):
+        start_date = self.decoded('DTSTART')
+        if isinstance(start_date, datetime.datetime):
+            return {
+                'dateTime': self._datetime_str_prop('DTSTART')
+            }
+        else:
+            if isinstance(start_date, datetime.date):
+                return {
+                    'date': start_date.isoformat()
+                }
+            raise ValueError('DTSTART must be date or datetime')
+
+    def _gcal_end(self):
+        if 'DTEND' in self:
+            end_date = self.decoded('DTEND')
+            if isinstance(end_date, datetime.datetime):
+                return {
+                    'dateTime': self._datetime_str_prop('DTEND')
+                }
+            else:
+                if isinstance(end_date, datetime.date):
+                    return {
+                        'date': end_date.isoformat()
+                    }
+                raise ValueError('DTEND must be date or datetime')
+        else:
+            if 'DURATION' in self:
+                start_date = self.decoded('DTSTART')
+                duration = self.decoded('DURATION')
+                end_date = start_date + duration
+
+                if isinstance(start_date, datetime.datetime):
+                    return {
+                        'dateTime': utc.normalize(end_date.astimezone(utc)).replace(tzinfo=None, microsecond=1).isoformat() + 'Z'
+                    }
+                else:
+                    if isinstance(start_date, datetime.date):
+                        return {
+                            'date': datetime.date(end_date.year, end_date.month, end_date.day).isoformat()
+                        }
+            raise ValueError('no DTEND or DURATION')
+        raise ValueError('end date/time not found')
+
+    def _put_to_gcal(self, gcal_event, prop, func, ics_prop=None):
+        if not ics_prop:
+            ics_prop = prop
+        if ics_prop in self:
+            gcal_event[prop] = func(ics_prop)
+
+    def to_gcal(self):
+        """Convert
+
+        Returns:
+            dict - google calendar#event resource
+        """
+
+        event = {
+            'iCalUID': self._str_prop('UID')
+        }
+
+        event['start'] = self._gcal_start()
+        event['end'] = self._gcal_end()
+
+        self._put_to_gcal(event, 'summary', self._str_prop)
+        self._put_to_gcal(event, 'description', self._str_prop)
+        self._put_to_gcal(event, 'location', self._str_prop)
+        self._put_to_gcal(event, 'created', self._datetime_str_prop)
+        self._put_to_gcal(
+            event, 'updated', self._datetime_str_prop, 'LAST-MODIFIED')
+        self._put_to_gcal(
+            event, 'transparency', lambda prop: self._str_prop(prop).lower(), 'TRANSP')
+
+        return event
+
+
+class CalendarConverter():
+    """Convert icalendar events to google calendar resources
+    """
+
+    logger = logging.getLogger('CalendarConverter')
+
+    def __init__(self, calendar=None):
+        self.calendar = calendar
+
+    def load(self, filename):
+        """ load calendar from ics file 
+        """
+        with open(filename, 'r', encoding='utf-8') as f:
+            self.calendar = Calendar.from_ical(f.read())
+            self.logger.info('%s loaded', filename)
+
+    def events_to_gcal(self):
+        """Convert events to google calendar resources
+        """
+
+        ics_events = self.calendar.walk(name='VEVENT')
+        self.logger.info('%d events readed', len(ics_events))
+
+        result = list(
+            map(lambda event: EventConverter(event).to_gcal(), ics_events))
+        self.logger.info('%d events converted', len(result))
+        return result

gcal_sync/sync.py

@@ -0,0 +1,124 @@
+import dateutil.parser
+import logging
+import operator
+
+
+class CalendarSync():
+    logger = logging.getLogger('CalendarSync')
+
+    def __init__(self, gcalendar, converter):
+        self.gcalendar = gcalendar
+        self.converter = converter
+
+    def _events_list_compare(self, items_src, items_dst, key='iCalUID'):
+        """ compare list of events by key
+
+        Arguments:
+            items_src {list of dict} -- source events
+            items_dst {list of dict} -- dest events
+            key {str} -- name of key to compare (default: {'iCalUID'})
+
+        Returns:
+            tuple -- (items_to_insert, 
+                      items_to_update, 
+                      items_to_delete)
+        """
+
+        def get_key(item): return item[key]
+
+        keys_src = list(map(get_key, items_src))
+        keys_dst = list(map(get_key, items_dst))
+
+        keys_to_insert = set(keys_src) - set(keys_dst)
+        keys_to_update = set(keys_src) & set(keys_dst)
+        keys_to_delete = set(keys_dst) - set(keys_src)
+
+        def items_by_keys(items, key_name, keys):
+            return list(filter(lambda item: item[key_name] in keys, items))
+
+        items_to_insert = items_by_keys(items_src, key, keys_to_insert)
+        items_to_update = list(zip(items_by_keys(
+            items_src, key, keys_to_update), items_by_keys(items_dst, key, keys_to_update)))
+        items_to_delete = items_by_keys(items_dst, key, keys_to_delete)
+
+        return items_to_insert, items_to_update, items_to_delete
+
+    def _filter_events_to_update(self):
+        """ Отбор событий к обновлению, по дате обновления
+
+        Arguments:
+            events -- список кортежей к обновлению (новое, старое)
+
+        Returns:
+            список кортежей к обновлению (новое, старое)
+        """
+
+        def filter_updated(event_tuple):
+            new, old = event_tuple
+            return dateutil.parser.parse(new['updated']) > dateutil.parser.parse(old['updated'])
+
+        self.to_update = list(filter(filter_updated, self.to_update))
+
+    def _filter_events_by_date(self, events, date, op):
+        """ Отбор событий по дате обновления
+
+        Arguments:
+            events -- список событий к обновлению
+            date {datetime} -- дата для сравнения
+            op {operator} -- оператор сравнения
+
+        Returns:
+            список событий
+        """
+
+        def filter_by_date(event):
+            return op(dateutil.parser.parse(event['updated']), date)
+
+        return list(filter(filter_by_date, events))
+
+    def prepare_sync(self, start_date):
+        events_src = self.converter.events_to_gcal()
+        events_dst = self.gcalendar.list_events_from(start_date)
+
+        # разбитие тестовых событий на будующие и прошлые
+        events_src_pending = self._filter_events_by_date(
+            events_src, start_date, operator.ge)
+        events_src_past = self._filter_events_by_date(
+            events_src, start_date, operator.lt)
+
+        events_src = None
+
+        # первоначальное сравнение списков
+        self.to_insert, self.to_update, self.to_delete = self._events_list_compare(
+            events_src_pending, events_dst)
+
+        events_src_pending, events_dst = None, None
+
+        # сравнение списка на удаление со списком прошлых событий, для определения доп событий к обновлению
+        _, add_to_update, self.to_delete = self._events_list_compare(
+            events_src_past, self.to_delete)
+        self.to_update.extend(add_to_update)
+
+        events_src_past = None
+
+        # проверка списка к вставке и перемещение доп. элементов в список к обновлению
+        add_to_update, self.to_insert = self.gcalendar.find_exists(
+            self.to_insert)
+        self.to_update.extend(add_to_update)
+
+        add_to_update = None
+
+        # отбор событий требующих обновления (по полю 'updated')
+        self._filter_events_to_update()
+
+        self.logger.info('prepared to sync: ( insert: %d, update: %d, delete: %d )',
+                         len(self.to_insert), len(self.to_update), len(self.to_delete))
+
+    def apply(self):
+        self.gcalendar.insert_events(self.to_insert)
+        self.gcalendar.update_events(self.to_update)
+        self.gcalendar.delete_events(self.to_delete)
+
+        self.logger.info('sync done')
+
+        self.to_insert, self.to_update, self.to_delete = [], [], []

gcal_sync/test-events.py

@@ -0,0 +1,42 @@
+import hashlib
+
+from pytz import UTC, timezone
+
+import datetime
+
+def sha1(string):
+    ''' Хеширование строки
+    '''
+    if isinstance(string, str):
+        string = string.encode('utf8')
+    h = hashlib.sha1()
+    h.update(string)
+    return h.hexdigest()
+
+def genenerate(count=10):
+    ''' Создание тестовых событий
+    '''
+    msk = timezone('Europe/Moscow')
+    now = UTC.localize(datetime.datetime.utcnow())
+    msk_now = msk.normalize(now.astimezone(msk))
+
+    one_hour = datetime.datetime(1,1,1,2) - datetime.datetime(1,1,1,1)
+    
+    start_time = msk_now - (one_hour * 3)
+    for i in range(count):
+        event_start = start_time + (one_hour * i)
+        event_end   = event_start + one_hour
+        updated = UTC.normalize(event_start.astimezone(UTC)).replace(tzinfo=None)
+        yield {
+            'summary': 'test event __ {}'.format(i),
+            'location': 'la la la {}'.format(i),
+            'description': 'test TEST -- test event {}'.format(i),
+            'start': {
+                'dateTime': event_start.isoformat()
+            },
+            'end': {
+                'dateTime': event_end.isoformat(),
+            },
+            "iCalUID": "{}@test-domain.ru".format(sha1("test - event {}".format(i))),
+            "updated": updated.isoformat() + 'Z',
+            "created": updated.isoformat() + 'Z'}

sample-config.yml

@@ -0,0 +1,7 @@
+
+#start_from: 2018-04-03T13:23:25.000001Z
+start_from: now
+service_account: service-account.json
+calendar:
+  google_id: google-calendar-id@group.calendar.google.com
+  source: my-test.ics

sync-calendar.py

@@ -0,0 +1,53 @@
+import yaml
+
+import dateutil.parser
+import datetime
+import logging
+import logging.config
+from gcal_sync import (
+    CalendarConverter,
+    GoogleCalendarService,
+    GoogleCalendar,
+    CalendarSync
+)
+
+
+def load_config():
+    with open('config.yml', 'r', encoding='utf-8') as f:
+        result = yaml.load(f)
+    return result
+
+
+def get_start_date(date_str):
+    result = datetime.datetime(1,1,1)
+    if 'now' == date_str:
+        result = datetime.datetime.utcnow()
+    else:
+        result = dateutil.parser.parse(date_str)
+    return result
+
+
+def main():
+    config = load_config()
+
+    if 'logging' in config:
+        logging.config.dictConfig(config['logging'])
+
+    calendarId = config['calendar']['google_id']
+    ics_filepath = config['calendar']['source']
+    srv_acc_file = config['service_account']
+
+    start = get_start_date(config['start_from'])
+
+    converter = CalendarConverter()
+    converter.load(ics_filepath)
+
+    service = GoogleCalendarService.from_srv_acc_file(srv_acc_file)
+    gcalendar = GoogleCalendar(service, calendarId)
+
+    sync = CalendarSync(gcalendar, converter)
+    sync.prepare_sync(start)
+    sync.apply()
+
+if __name__ == '__main__':
+    main()