View Issue Details

IDProjectCategoryView StatusLast Update
0006296Tine 2.0Calendarpublic2017-11-09 17:48
ReporterweberhoAssigned Tocweiss 
PrioritynormalSeverityminorReproducibilityalways
Status resolvedResolutionfixed 
Platformx86_64OSOpenSUSEOS Version11.4
Product VersionMilan (2012-03-2) 
Target VersionKristina (2013.03.6)Fixed in VersionKristina (2013.03.6) 
Summary0006296: Calendar.import shifts dates upon import from ics files
DescriptionSome records from an ics file with birthdays are no imported correctly. Instead of "1970-10-25" the example is imported as "16.08.2012 start at 10:33".

I'm using the "php tine20.php -v --method=Calendar.import ... -v plugin=Calendar_Import_Ical" procedure.
Additional InformationExample ics-file:

BEGIN:VCALENDAR
PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN
VERSION:2.0
X-WR-CALNAME:geburtstage
X-WR-TIMEZONE:Europe/Berlin
BEGIN:VTIMEZONE
TZID:Africa/Ceuta
X-LIC-LOCATION:Africa/Ceuta
BEGIN:DAYLIGHT
TZOFFSETFROM:+0100
TZOFFSETTO:+0200
TZNAME:CEST
DTSTART:19700329T020000
RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3
END:DAYLIGHT
BEGIN:STANDARD
TZOFFSETFROM:+0200
TZOFFSETTO:+0100
TZNAME:CET
DTSTART:19701025T030000
RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
END:STANDARD
END:VTIMEZONE
BEGIN:VEVENT
CREATED:20070828T120758Z
LAST-MODIFIED:20070828T120857Z
DTSTAMP:20070828T120857Z
UID:4f7e9c9b-0f21-4ce6-9205-c5c844121b34
SUMMARY:Test-Birthday
PRIORITY:0
RRULE:FREQ=YEARLY
DTSTART:19650827
DTEND:19650828
CLASS:PUBLIC
END:VEVENT
END:VCALENDAR
TagsNo tags attached.
mwticket

Relationships

related to 0006158 resolvedcweiss fix ics import with VTIMEZONE 
related to 0006448 new use vcalendar converter for ics export and remove qcal stuff 
related to 0009856 resolvedpschuele ics import: recurring events one day earlier 
related to 0013526 gerritpschuele Calendar ICS import ignores local timezone and adds an organizer 

Activities

weberho

weberho

2012-05-10 21:40

developer  

tine20-ical-import.patch (4,463 bytes)
diff -ur tine20.orig/Calendar/Import/Ical.php tine20/Calendar/Import/Ical.php
--- tine20.orig/Calendar/Import/Ical.php	2012-04-12 13:32:09.000000000 +0200
+++ tine20/Calendar/Import/Ical.php	2012-05-10 17:35:55.366342509 +0200
@@ -100,14 +100,6 @@
      */
     public function import($_resource = NULL, $_clientRecordData = array())
     {
-        // force correct line ends
-        require_once 'StreamFilter/StringReplace.php';
-        $filter = stream_filter_append($_resource, 'str.replace', STREAM_FILTER_READ, array(
-            'search'            => '/(?<!\r)\n/',
-            'replace'           => "\r\n",
-            'searchIsRegExp'    => TRUE
-        ));
-        
         if (! $this->_options['importContainerId']) {
             throw new Tinebase_Exception_InvalidArgument('you need to define a importContainerId');
         }
@@ -202,6 +194,7 @@
                         break;
                     case 'DATE-TIME':
                         $value = new Tinebase_DateTime($icalValue->getValue(), $eventData['originator_tz']);
+			break;
                     case 'TEXT':
                         $value = str_replace(array('\\,', '\\n'), array(',', "\n"), $icalValue->getValue());
                         break;
diff -ur tine20.orig/library/qCal/lib/qCal/Component.php tine20/library/qCal/lib/qCal/Component.php
--- tine20.orig/library/qCal/lib/qCal/Component.php	2012-04-12 13:32:16.000000000 +0200
+++ tine20/library/qCal/lib/qCal/Component.php	2012-05-10 21:22:19.873464994 +0200
@@ -239,6 +239,7 @@
 		$component->setParent($this);
 		// make sure if a timezone is requested that it is available...
 		$timezones = $this->getTimezones(); 
+		$tzids = array_keys($timezones);
 		// we only need to check if tzid exists if we are attaching something other than a timezone...
 		if (!($component instanceof qCal_Component_Vtimezone)) {
 			foreach ($component->getProperties() as $pname => $properties) {
@@ -258,7 +259,7 @@
 						switch ($param) {
 							case "TZID":
 								$tzid = strtoupper($val);
-								if (!array_key_exists($tzid, $timezones)) {
+								if (!in_array($tzid, $tzids)) {
 									throw new qCal_Exception_MissingComponent('TZID "' . $tzid . '" not defined');
 								}
 								break;
@@ -582,4 +583,4 @@
 	
 	}
 
-}
\ No newline at end of file
+}
diff -ur tine20.orig/library/qCal/lib/qCal/DateTime.php tine20/library/qCal/lib/qCal/DateTime.php
--- tine20.orig/library/qCal/lib/qCal/DateTime.php	2012-04-12 13:32:16.000000000 +0200
+++ tine20/library/qCal/lib/qCal/DateTime.php	2012-05-10 17:40:33.635566326 +0200
@@ -65,7 +65,13 @@
 	 * @todo Should this accept qCal_Date and qCal_DateTime objects?
 	 */
 	public static function factory($datetime, $timezone = null) {
-	
+		if (substr($datetime,0,4)=='TZID') {
+			// Handle DateTime with Timezones
+			$buffer = explode(';', $datetime, 2);
+			$datetime = $buffer[1];
+			$buffer = explode('=', $buffer[0], 2);
+			$timezone = $buffer[1];
+		}	
 		if (is_null($timezone) || !($timezone instanceof qCal_Timezone)) {
 			// @todo Make sure this doesn't cause any issues 
 			// detect if we're working with a UTC string like "19970101T180000Z", where the Z means use UTC time
@@ -249,4 +255,4 @@
 	
 	}
 
-}
\ No newline at end of file
+}
diff -ur tine20.orig/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php tine20/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php
--- tine20.orig/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php	2012-04-12 13:32:16.000000000 +0200
+++ tine20/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php	2012-05-10 17:44:49.576370974 +0200
@@ -15,11 +15,6 @@
 class qCal_Parser_Lexer_iCalendar extends qCal_Parser_Lexer {
 	
 	/**
-	 * @var string character(s) used to terminate lines
-	 */
-	protected $line_terminator;
-	
-	/**
 	 * Constructor 
 	 * @param string The iCalendar data to be parsed
 	 * @access public
@@ -27,7 +22,6 @@
 	public function __construct($content) {
 	
 		parent::__construct($content);
-		$this->line_terminator = chr(13) . chr(10);
 	
 	}
 	/**
@@ -121,7 +115,9 @@
 	protected function unfold($content) {
 	
 		$return = array();
-		$lines = explode($this->line_terminator, $content);
+		// handle linux AND windows line-breaks!
+		$content = str_replace("\r", "\n", $content);
+		$lines = explode("\n", $content);
 		foreach ($lines as $line) {
 			$checkempty = trim($line);
 			if (empty($checkempty)) continue;
@@ -138,4 +134,4 @@
 	
 	}
 
-}
\ No newline at end of file
+}
tine20-ical-import.patch (4,463 bytes)
weberho

weberho

2012-05-10 21:40

developer   ~0007480

Last edited: 2012-05-10 21:41

View 2 revisions

I have finally prepeared a patch, which allows import of several calendars generated with the latest lightning version. A total of around 3000 meetings, birthdays and holidays could successfully be imported.

The patch should fix for several ical-import-issues including 0006158;

Patch description:

1) Fixed qCal to handle Dos and Unix linebreaks
2) Fix qCal bug in timezone handling
3) Fixed a bug that caused tine20 to ignore the timezone when importing records

pschuele

pschuele

2012-05-13 21:56

administrator   ~0007520

pushed the patch to gerrit for review: https://gerrit.tine20.org/tine20/#/c/615/
pschuele

pschuele

2012-05-14 09:24

administrator   ~0007522

the jenkins build failed, looks like the timezone is not taken into account:

phpunit-int:
     [exec] PHPUnit 3.4.5 by Sebastian Bergmann.
     [exec]
     [exec] ............................................................ 60 / 1082
     [exec] .............S.............................................. 120 / 1082
     [exec] ............................................................ 180 / 1082
     [exec] ..................................S......................... 240 / 1082
     [exec] SSSSSSSSSSSSSSSS..........SSSSSSSSSSSSSSSSSSSSS............. 300 / 1082
     [exec] ............................................................ 360 / 1082
     [exec] ............................................................ 420 / 1082
     [exec] ..............................................S............. 480 / 1082
     [exec] ...............................S............................ 540 / 1082
     [exec] ............SS.............................................. 600 / 1082
     [exec] ............................................................ 660 / 1082
     [exec] .........SSSSSSSSSSSSSS..................................... 720 / 1082
     [exec] .....FF........................S............................ 780 / 1082
     [exec] .......................................S.................... 840 / 1082
     [exec] ............................................................ 900 / 1082
     [exec] ............................................................ 960 / 1082
     [exec] ......S....S...S.........I...........SSS.......S..I......SSS 1020 / 1082
     [exec] S........................................................... 1080 / 1082
     [exec] ..
     [exec]
     [exec] Time: 15:09, Memory: 313.25Mb
     [exec]
     [exec] There were 2 failures:
     [exec]
     [exec] 1) Calendar_Import_ICalTest::testImportSimpleFromString
     [exec] Failed asserting that two strings are equal.
     [exec] --- Expected
     [exec] +++ Actual
     [exec] @@ @@
     [exec] -2008-11-05 15:00:00
     [exec] +2008-11-05 14:00:00
     [exec]
     [exec]
     [exec] 2) Calendar_Import_ICalTest::testImportSimpleFromFile
     [exec] Failed asserting that two strings are equal.
     [exec] --- Expected
     [exec] +++ Actual
     [exec] @@ @@
     [exec] -2008-11-05 15:00:00
     [exec] +2008-11-05 14:00:00
     [exec]
     [exec]
     [exec] FAILURES!
     [exec] Tests: 1082, Assertions: 4381, Failures: 2, Incomplete: 2, Skipped: 70.
weberho

weberho

2012-05-14 12:03

developer   ~0007530

This is strange. Input value is "DTSTART:20081105T150000Z" which means UTC; What timezones are used by the jenkins tool?

When I do a manual import of event 3632597 here at my system, I see the meeting at 16:00, which is correct, because in November it's Standard Central Europe Time (UTC+1); "2008-11-05 15:00:00" is correctly inserted into the database, too.

I'm quite sure, that the difference is caused by the fix on Calendar/Import/Ical.php line 204, which caused the date/time conversion to be ignored.
weberho

weberho

2012-05-14 13:04

developer   ~0007532

Phillip, can you please tell me, what timezone you expect to be used for output within the failing tests!?
pschuele

pschuele

2012-05-14 13:22

administrator   ~0007534

Last edited: 2012-05-14 13:26

View 3 revisions

the timezone should be UTC, too.

here is the file, that is used by the test, event uid 3632597 / dtstart is tested. i think, the test is working correctly as UTC should be default for imports.





BEGIN:VCALENDAR
PRODID:-//Microsoft Corporation//Outlook 12.0 MIMEDIR//EN
VERSION:2.0
METHOD:PUBLISH
BEGIN:VEVENT
CLASS:PUBLIC
CREATED:20081105T104258Z
DESCRIPTION:A discussion with Tyler about how everything is over coffee
  and donuts. I imagine we'll probably talk about our personal projects.
DTEND:20081105T160000Z
DTSTAMP:20081105T104200Z
DTSTART:20081105T150000Z
LAST-MODIFIED:20081105T104258Z
SEQUENCE:0
SUMMARY:Meeting with Tyler at Starbucks
TRANSP:OPAQUE
UID:3632597
END:VEVENT
END:VCALENDAR

weberho

weberho

2012-05-14 13:46

developer   ~0007536

I have found that; importing that in my local installation gives (as explained above) the correct entry; Therefore I'm not sure, what happens here...
pschuele

pschuele

2012-08-29 17:56

administrator   ~0008816

see https://gerrit.tine20.org/tine20/#/c/615/

but i think we won't fix this as qcal is going to be replaced soon.
weberho

weberho

2013-05-23 13:30

developer   ~0011120

@pschuele: You have merged most of my changes with

https://gerrit.tine20.org/tine20/#/c/2019/
https://gerrit.tine20.org/tine20/#/c/1997/

There are still some outstanding issues I have fixed in the patch, which I upload in a newer version here.
weberho

weberho

2013-05-23 13:31

developer  

tine20-ical-import-2.patch (3,183 bytes)
diff -ur t.orig/library/qCal/lib/qCal/Component.php t/library/qCal/lib/qCal/Component.php
--- t.orig/library/qCal/lib/qCal/Component.php	2013-05-23 10:59:03.000000000 +0200
+++ t/library/qCal/lib/qCal/Component.php	2013-05-23 12:54:41.042147743 +0200
@@ -239,6 +239,7 @@
 		$component->setParent($this);
 		// make sure if a timezone is requested that it is available...
 		$timezones = $this->getTimezones(); 
+		$tzids = array_keys($timezones);
 		// we only need to check if tzid exists if we are attaching something other than a timezone...
 		if (!($component instanceof qCal_Component_Vtimezone)) {
 			foreach ($component->getProperties() as $pname => $properties) {
@@ -258,7 +259,7 @@
 						switch ($param) {
 							case "TZID":
 								$tzid = strtoupper($val);
-								if (!array_key_exists($tzid, $timezones)) {
+								if (!in_array($tzid, $tzids)) {
 									throw new qCal_Exception_MissingComponent('TZID "' . $tzid . '" not defined');
 								}
 								break;
@@ -582,4 +583,4 @@
 	
 	}
 
-}
\ Kein Zeilenumbruch am Dateiende.
+}
diff -ur t.orig/library/qCal/lib/qCal/DateTime.php t/library/qCal/lib/qCal/DateTime.php
--- t.orig/library/qCal/lib/qCal/DateTime.php	2013-05-23 10:59:03.000000000 +0200
+++ t/library/qCal/lib/qCal/DateTime.php	2013-05-23 12:54:41.054147970 +0200
@@ -65,7 +65,13 @@
 	 * @todo Should this accept qCal_Date and qCal_DateTime objects?
 	 */
 	public static function factory($datetime, $timezone = null) {
-	
+		if (substr($datetime,0,4)=='TZID') {
+			// Handle DateTime with Timezones
+			$buffer = explode(';', $datetime, 2);
+			$datetime = $buffer[1];
+			$buffer = explode('=', $buffer[0], 2);
+			$timezone = $buffer[1];
+		}	
 		if (is_null($timezone) || !($timezone instanceof qCal_Timezone)) {
 			// @todo Make sure this doesn't cause any issues 
 			// detect if we're working with a UTC string like "19970101T180000Z", where the Z means use UTC time
@@ -249,4 +255,4 @@
 	
 	}
 
-}
\ Kein Zeilenumbruch am Dateiende.
+}
diff -ur t.orig/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php t/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php
--- t.orig/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php	2013-05-23 10:59:03.000000000 +0200
+++ t/library/qCal/lib/qCal/Parser/Lexer/iCalendar.php	2013-05-23 12:54:41.062148124 +0200
@@ -15,11 +15,6 @@
 class qCal_Parser_Lexer_iCalendar extends qCal_Parser_Lexer {
 	
 	/**
-	 * @var string character(s) used to terminate lines
-	 */
-	protected $line_terminator;
-	
-	/**
 	 * Constructor 
 	 * @param string The iCalendar data to be parsed
 	 * @access public
@@ -27,7 +22,6 @@
 	public function __construct($content) {
 	
 		parent::__construct($content);
-		$this->line_terminator = chr(13) . chr(10);
 	
 	}
 	/**
@@ -121,7 +115,9 @@
 	protected function unfold($content) {
 	
 		$return = array();
-		$lines = explode($this->line_terminator, $content);
+		// handle linux AND windows line-breaks!
+		$content = str_replace("\r", "\n", $content);
+		$lines = explode("\n", $content);
 		foreach ($lines as $line) {
 			$checkempty = trim($line);
 			if (empty($checkempty)) continue;
@@ -138,4 +134,4 @@
 	
 	}
 
-}
\ Kein Zeilenumbruch am Dateiende.
+}
cweiss

cweiss

2013-07-17 09:54

administrator   ~0011652

ical is now based on converters, isshue should be solved.

if not, please reopen
pschuele

pschuele

2017-11-09 17:48

administrator   ~0021118

merged the patch.

and we still use qcal for imports :( see 0006448

Issue History

Date Modified Username Field Change
2012-04-20 17:38 weberho New Issue
2012-04-20 21:06 pschuele Target Version => Milan (2012-03-3)
2012-04-20 21:06 pschuele Relationship added related to 0006158
2012-05-10 21:40 weberho File Added: tine20-ical-import.patch
2012-05-10 21:40 weberho Note Added: 0007480
2012-05-10 21:41 weberho Note Edited: 0007480 View Revisions
2012-05-11 10:51 pschuele Assigned To => pschuele
2012-05-11 10:51 pschuele Status new => assigned
2012-05-13 21:56 pschuele Note Added: 0007520
2012-05-13 21:56 pschuele Status assigned => gerrit
2012-05-14 09:24 pschuele Note Added: 0007522
2012-05-14 10:05 pschuele Assigned To pschuele => weberho
2012-05-14 10:05 pschuele Status gerrit => assigned
2012-05-14 12:03 weberho Note Added: 0007530
2012-05-14 13:03 weberho Assigned To weberho => pschuele
2012-05-14 13:04 weberho Note Added: 0007532
2012-05-14 13:22 pschuele Note Added: 0007534
2012-05-14 13:26 pschuele Note Edited: 0007534 View Revisions
2012-05-14 13:26 pschuele Note Edited: 0007534 View Revisions
2012-05-14 13:27 pschuele Status assigned => feedback
2012-05-14 13:46 weberho Note Added: 0007536
2012-05-14 13:46 weberho Status feedback => assigned
2012-05-14 14:33 pschuele Assigned To pschuele => cweiss
2012-05-14 14:33 pschuele Target Version Milan (2012-03-3) => Milan (2012.03.4)
2012-06-26 11:51 pschuele Target Version Milan (2012.03.4) => Milan (2012.03.5)
2012-06-27 11:06 pschuele Target Version Milan (2012.03.5) => Milan (2012.03.6)
2012-08-22 12:42 pschuele Target Version Milan (2012.03.6) => Joey (2012.10.1~beta1)
2012-08-29 16:52 pschuele Relationship added related to 0006448
2012-08-29 17:56 pschuele Note Added: 0008816
2012-09-11 15:40 pschuele Target Version Joey (2012.10.1~beta1) => Joey (2012.10.1~beta2)
2012-09-20 19:34 pschuele Target Version Joey (2012.10.1~beta2) => Joey (2012.10) RC 1
2012-09-28 11:42 cweiss Target Version Joey (2012.10) RC 1 => Kristina (2013.03.1~alpha1)
2013-02-06 13:23 pschuele Target Version Kristina (2013.03.1~alpha1) => Kristina (2013.03.1~beta1)
2013-02-14 11:27 pschuele Target Version Kristina (2013.03.1~beta1) => Kristina (2013.03.1~rc1)
2013-02-20 14:29 pschuele Target Version Kristina (2013.03.1~rc1) => Kristina (2013.03.2)
2013-02-22 13:01 cweiss Severity major => minor
2013-03-19 13:20 pschuele Target Version Kristina (2013.03.2) => known bugs
2013-05-23 13:30 weberho Note Added: 0011120
2013-05-23 13:31 weberho File Added: tine20-ical-import-2.patch
2013-07-17 09:54 cweiss Note Added: 0011652
2013-07-17 09:54 cweiss Status assigned => resolved
2013-07-17 09:54 cweiss Fixed in Version => Kristina (2013.03.6)
2013-07-17 09:54 cweiss Resolution open => fixed
2013-07-17 09:58 cweiss Target Version known bugs => Kristina (2013.03.6)
2014-05-15 11:35 pschuele Relationship added related to 0009856
2017-10-13 09:11 pschuele Relationship added related to 0013526
2017-11-09 17:48 pschuele Note Added: 0021118