fix daterange filter for multi-timezone setups (#1321)

README.md

@@ -21,7 +21,7 @@ It is built with modern technologies such as Symfony, Bootstrap, RESTful API, Do
 ### Requirements
 
 - PHP 7.2 or higher
-- Database (MySQL, MariaDB, SQLite)
+- Database (MySQL/MariaDB with timezone data, SQLite for development) 
 - Webserver (nginx, Apache)
 - A modern browser
 - [Other libraries](https://www.kimai.org/download/)

UPGRADING.md

@@ -8,6 +8,11 @@ you can upgrade your Kimai installation to the latest stable release.
 Check below if there are more version specific steps required, which need to be executed after the normal update process.
 Perform EACH version specific task between your version and the new one, otherwise you risk data inconsistency or a broken installation.
 
+## [1.7](https://github.com/kevinpapst/kimai2/releases/tag/1.7)
+
+New feature requires the timezone date within MySQL/MariaDB. 
+This is especially important for self-hosted systems [where you need to import the data manually](https://mariadb.com/kb/en/library/mysql_tzinfo_to_sql/). 
+
 ## [1.6](https://github.com/kevinpapst/kimai2/releases/tag/1.6), [1.6.1](https://github.com/kevinpapst/kimai2/releases/tag/1.6.1), [1.6.2](https://github.com/kevinpapst/kimai2/releases/tag/1.6.2)
 
 **New database tables and fields were created, don't forget to [run the updater](https://www.kimai.org/documentation/updates.html).**

config/packages/doctrine/mysql.yaml

@@ -0,0 +1,119 @@
+doctrine:
+     orm:
+         dql:
+             datetime_functions:
+                 addtime: DoctrineExtensions\Query\Mysql\AddTime
+                 convert_tz: DoctrineExtensions\Query\Mysql\ConvertTz
+                 date: DoctrineExtensions\Query\Mysql\Date
+                 date_format: DoctrineExtensions\Query\Mysql\DateFormat
+                 dateadd: DoctrineExtensions\Query\Mysql\DateAdd
+                 datesub: DoctrineExtensions\Query\Mysql\DateSub
+                 datediff: DoctrineExtensions\Query\Mysql\DateDiff
+                 day: DoctrineExtensions\Query\Mysql\Day
+                 dayname: DoctrineExtensions\Query\Mysql\DayName
+                 dayofweek: DoctrineExtensions\Query\Mysql\DayOfWeek
+                 dayofyear: DoctrineExtensions\Query\Mysql\DayOfYear
+                 div: DoctrineExtensions\Query\Mysql\Div
+                 from_unixtime: DoctrineExtensions\Query\Mysql\FromUnixtime
+                 hour: DoctrineExtensions\Query\Mysql\Hour
+                 last_day: DoctrineExtensions\Query\Mysql\LastDay
+                 makedate: DoctrineExtensions\Query\Mysql\MakeDate
+                 minute: DoctrineExtensions\Query\Mysql\Minute
+                 now: DoctrineExtensions\Query\Mysql\Now
+                 month: DoctrineExtensions\Query\Mysql\Month
+                 monthname: DoctrineExtensions\Query\Mysql\MonthName
+                 period_diff: DoctrineExtensions\Query\Mysql\PeriodDiff
+                 second: DoctrineExtensions\Query\Mysql\Second
+                 sectotime: DoctrineExtensions\Query\Mysql\SecToTime
+                 strtodate: DoctrineExtensions\Query\Mysql\StrToDate
+                 time: DoctrineExtensions\Query\Mysql\Time
+                 timediff: DoctrineExtensions\Query\Mysql\TimeDiff
+                 timestampadd: DoctrineExtensions\Query\Mysql\TimestampAdd
+                 timestampdiff: DoctrineExtensions\Query\Mysql\TimestampDiff
+                 timetosec: DoctrineExtensions\Query\Mysql\TimeToSec
+                 week: DoctrineExtensions\Query\Mysql\Week
+                 weekday: DoctrineExtensions\Query\Mysql\WeekDay
+                 year: DoctrineExtensions\Query\Mysql\Year
+                 yearmonth: DoctrineExtensions\Query\Mysql\YearMonth
+                 yearweek: DoctrineExtensions\Query\Mysql\YearWeek
+                 unix_timestamp: DoctrineExtensions\Query\Mysql\UnixTimestamp
+                 utc_timestamp: DoctrineExtensions\Query\Mysql\UtcTimestamp
+                 extract: DoctrineExtensions\Query\Mysql\Extract
+
+             numeric_functions:
+                 acos: DoctrineExtensions\Query\Mysql\Acos
+                 asin: DoctrineExtensions\Query\Mysql\Asin
+                 atan2: DoctrineExtensions\Query\Mysql\Atan2
+                 atan: DoctrineExtensions\Query\Mysql\Atan
+                 bit_count: DoctrineExtensions\Query\Mysql\BitCount
+                 bit_xor: DoctrineExtensions\Query\Mysql\BitXor
+                 ceil: DoctrineExtensions\Query\Mysql\Ceil
+                 cos: DoctrineExtensions\Query\Mysql\Cos
+                 cot: DoctrineExtensions\Query\Mysql\Cot
+                 degrees: DoctrineExtensions\Query\Mysql\Degrees
+                 exp: DoctrineExtensions\Query\Mysql\Exp
+                 floor: DoctrineExtensions\Query\Mysql\Floor
+                 log: DoctrineExtensions\Query\Mysql\Log
+                 log10: DoctrineExtensions\Query\Mysql\Log10
+                 log2: DoctrineExtensions\Query\Mysql\Log2
+                 pi: DoctrineExtensions\Query\Mysql\Pi
+                 power: DoctrineExtensions\Query\Mysql\Power
+                 quarter: DoctrineExtensions\Query\Mysql\Quarter
+                 radians: DoctrineExtensions\Query\Mysql\Radians
+                 rand: DoctrineExtensions\Query\Mysql\Rand
+                 round: DoctrineExtensions\Query\Mysql\Round
+                 stddev: DoctrineExtensions\Query\Mysql\StdDev
+                 sin: DoctrineExtensions\Query\Mysql\Sin
+                 std: DoctrineExtensions\Query\Mysql\Std
+                 tan: DoctrineExtensions\Query\Mysql\Tan
+                 variance: DoctrineExtensions\Query\Mysql\Variance
+
+             string_functions:
+                 aes_decrypt: DoctrineExtensions\Query\Mysql\AesDecrypt
+                 aes_encrypt: DoctrineExtensions\Query\Mysql\AesEncrypt
+                 any_value: DoctrineExtensions\Query\Mysql\AnyValue
+                 ascii: DoctrineExtensions\Query\Mysql\Ascii
+                 binary: DoctrineExtensions\Query\Mysql\Binary
+                 cast: DoctrineExtensions\Query\Mysql\Cast
+                 char_length: DoctrineExtensions\Query\Mysql\CharLength
+                 collate: DoctrineExtensions\Query\Mysql\Collate
+                 concat_ws: DoctrineExtensions\Query\Mysql\ConcatWs
+                 countif: DoctrineExtensions\Query\Mysql\CountIf
+                 crc32: DoctrineExtensions\Query\Mysql\Crc32
+                 degrees: DoctrineExtensions\Query\Mysql\Degrees
+                 field: DoctrineExtensions\Query\Mysql\Field
+                 find_in_set: DoctrineExtensions\Query\Mysql\FindInSet
+                 format: DoctrineExtensions\Query\Mysql\Format
+                 greatest: DoctrineExtensions\Query\Mysql\Greatest
+                 group_concat: DoctrineExtensions\Query\Mysql\GroupConcat
+                 hex: DoctrineExtensions\Query\Mysql\Hex
+                 ifelse: DoctrineExtensions\Query\Mysql\IfElse
+                 ifnull: DoctrineExtensions\Query\Mysql\IfNull
+                 inet_aton: DoctrineExtensions\Query\Mysql\InetAton
+                 inet_ntoa: DoctrineExtensions\Query\Mysql\InetNtoa
+                 inet6_aton: DoctrineExtensions\Query\Mysql\Inet6Aton
+                 inet6_ntoa: DoctrineExtensions\Query\Mysql\Inet6Ntoa
+                 instr: DoctrineExtensions\Query\Mysql\Instr
+                 is_ipv4: DoctrineExtensions\Query\Mysql\IsIpv4
+                 is_ipv4_compat: DoctrineExtensions\Query\Mysql\IsIpv4Compat
+                 is_ipv4_mapped: DoctrineExtensions\Query\Mysql\IsIpv4Mapped
+                 is_ipv6: DoctrineExtensions\Query\Mysql\IsIpv6
+                 lag: DoctrineExtensions\Query\Mysql\Lag
+                 lead: DoctrineExtensions\Query\Mysql\Lead
+                 least: DoctrineExtensions\Query\Mysql\Least
+                 lpad: DoctrineExtensions\Query\Mysql\Lpad
+                 match: DoctrineExtensions\Query\Mysql\MatchAgainst
+                 md5: DoctrineExtensions\Query\Mysql\Md5
+                 nullif: DoctrineExtensions\Query\Mysql\NullIf
+                 over: DoctrineExtensions\Query\Mysql\Over
+                 radians: DoctrineExtensions\Query\Mysql\Radians
+                 regexp: DoctrineExtensions\Query\Mysql\Regexp
+                 replace: DoctrineExtensions\Query\Mysql\Replace
+                 rpad: DoctrineExtensions\Query\Mysql\Rpad
+                 sha1: DoctrineExtensions\Query\Mysql\Sha1
+                 sha2: DoctrineExtensions\Query\Mysql\Sha2
+                 soundex: DoctrineExtensions\Query\Mysql\Soundex
+                 str_to_date: DoctrineExtensions\Query\Mysql\StrToDate
+                 substring_index: DoctrineExtensions\Query\Mysql\SubstringIndex
+                 unhex: DoctrineExtensions\Query\Mysql\Unhex
+                 uuid_short: DoctrineExtensions\Query\Mysql\UuidShort

config/packages/doctrine/sqlite.yaml

@@ -0,0 +1,56 @@
+doctrine:
+     orm:
+         dql:
+             datetime_functions:
+                 convert_tz: App\Doctrine\Query\Sqlite\ConvertTz
+                 date: DoctrineExtensions\Query\Sqlite\Date
+                 date_format: DoctrineExtensions\Query\Sqlite\DateFormat
+                 #dateadd: DoctrineExtensions\Query\Sqlite\DateAdd
+                 #datediff: DoctrineExtensions\Query\Sqlite\DateDiff
+                 day: DoctrineExtensions\Query\Sqlite\Day
+                 dayname: DoctrineExtensions\Query\Sqlite\DayName
+                 hour: DoctrineExtensions\Query\Sqlite\Hour
+                 julianday: DoctrineExtensions\Query\Sqlite\JulianDay
+                 minute: DoctrineExtensions\Query\Sqlite\Minute
+                 #month: DoctrineExtensions\Query\Sqlite\Month
+                 #monthname: DoctrineExtensions\Query\Sqlite\MonthName
+                 second: DoctrineExtensions\Query\Sqlite\Second
+                 month:  DoctrineExtensions\Query\Sqlite\Month
+                 strftime:  DoctrineExtensions\Query\Sqlite\StrfTime
+                 #strtodate: DoctrineExtensions\Query\Sqlite\StrToDate
+                 #time: DoctrineExtensions\Query\Sqlite\Time
+                 #timestampadd: DoctrineExtensions\Query\Sqlite\TimestampAdd
+                 #timestampdiff: DoctrineExtensions\Query\Sqlite\TimestampDiff
+                 week: DoctrineExtensions\Query\Sqlite\Week
+                 weekday: DoctrineExtensions\Query\Sqlite\WeekDay
+                 year: DoctrineExtensions\Query\Sqlite\Year
+
+
+             string_functions:
+#                 binary: DoctrineExtensions\Query\Sqlite\Binary
+#                 char_length: DoctrineExtensions\Query\Sqlite\CharLength
+                 concat_ws: DoctrineExtensions\Query\Sqlite\ConcatWs
+#                 countif: DoctrineExtensions\Query\Sqlite\CountIf
+#                 crc32: DoctrineExtensions\Query\Sqlite\Crc32
+#                 degrees: DoctrineExtensions\Query\Sqlite\Degrees
+#                 field: DoctrineExtensions\Query\Sqlite\Field
+#                 find_in_set: DoctrineExtensions\Query\Sqlite\FindInSet
+#                 group_concat: DoctrineExtensions\Query\Sqlite\GroupConcat
+                 greatest: DoctrineExtensions\Query\Sqlite\Greatest
+                 ifelse: DoctrineExtensions\Query\Sqlite\IfElse
+                 ifnull: DoctrineExtensions\Query\Sqlite\IfNull
+                 least: DoctrineExtensions\Query\Sqlite\Least
+#                 match: DoctrineExtensions\Query\Sqlite\MatchAgainst
+#                 md5: DoctrineExtensions\Query\Sqlite\Md5
+#                 nullif: DoctrineExtensions\Query\Sqlite\NullIf
+#                 radians: DoctrineExtensions\Query\Sqlite\Radians
+                 random: DoctrineExtensions\Query\Sqlite\Random
+#                 regexp: DoctrineExtensions\Query\Sqlite\Regexp
+                 replace: DoctrineExtensions\Query\Sqlite\Replace
+#                 sha1: DoctrineExtensions\Query\Sqlite\Sha1
+#                 sha2: DoctrineExtensions\Query\Sqlite\Sha2
+#                 soundex: DoctrineExtensions\Query\Sqlite\Soundex
+#                 uuid_short: DoctrineExtensions\Query\Sqlite\UuidShort
+
+             numeric_functions:
+                 round: DoctrineExtensions\Query\Sqlite\Round

src/DependencyInjection/Compiler/DoctrineCompilerPass.php

@@ -70,18 +70,10 @@ class DoctrineCompilerPass implements CompilerPassInterface
         $engine = $this->findEngine();
 
         $configDir = realpath(
-            $container->getParameter('kernel.project_dir') . '/vendor/beberlei/doctrineextensions/config/'
+            $container->getParameter('kernel.project_dir') . '/config/packages/doctrine/'
         );
 
-        if (!file_exists($configDir)) {
+        $configFile = $configDir . '/' . $engine . '.yaml';
-            @trigger_error('Using deprecated doctrine extensions config directory', E_USER_DEPRECATED);
-
-            $configDir = realpath(
-                $container->getParameter('kernel.project_dir') . '/vendor/beberlei/DoctrineExtensions/config/'
-            );
-        }
-
-        $configFile = $configDir . '/' . $engine . '.yml';
 
         if (!file_exists($configFile)) {
             throw new \Exception('Could not find config file for database engine. Looked at ' . $configFile);

src/Doctrine/Query/Sqlite/ConvertTz.php

@@ -0,0 +1,57 @@
+<?php
+
+/*
+ * This file is part of the Kimai time-tracking app.
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace App\Doctrine\Query\Sqlite;
+
+use Doctrine\ORM\Query\AST\Functions\FunctionNode;
+use Doctrine\ORM\Query\Lexer;
+use Doctrine\ORM\Query\Parser;
+use Doctrine\ORM\Query\SqlWalker;
+
+/**
+ * This is just a fake, as SQLITE does not support this by now.
+ */
+class ConvertTz extends FunctionNode
+{
+    protected $dateExpression;
+
+    protected $fromTz;
+
+    protected $toTz;
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSql(SqlWalker $sqlWalker)
+    {
+        $fieldName = $sqlWalker->walkArithmeticExpression($this->dateExpression);
+        $fromTz = $sqlWalker->walkStringPrimary($this->fromTz);
+        $toTz = $sqlWalker->walkStringPrimary($this->toTz);
+
+        return sprintf('%s', $fieldName);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function parse(Parser $parser)
+    {
+        $parser->match(Lexer::T_IDENTIFIER);
+        $parser->match(Lexer::T_OPEN_PARENTHESIS);
+
+        $this->dateExpression = $parser->ArithmeticExpression();
+        $parser->match(Lexer::T_COMMA);
+
+        $this->fromTz = $parser->StringPrimary();
+        $parser->match(Lexer::T_COMMA);
+
+        $this->toTz = $parser->StringPrimary();
+        $parser->match(Lexer::T_CLOSE_PARENTHESIS);
+    }
+}

src/Repository/TimesheetRepository.php

@@ -218,14 +218,14 @@ class TimesheetRepository extends EntityRepository
 
         if (!empty($begin)) {
             $qb
-                ->andWhere($qb->expr()->gt('t.begin', ':from'))
+                ->andWhere($qb->expr()->gte($this->getDatetimeFieldSql('t.begin'), ':from'))
-                ->setParameter('from', $begin, Type::DATETIME);
+                ->setParameter('from', $begin);
         }
 
         if (!empty($end)) {
             $qb
-                ->andWhere($qb->expr()->lt('t.end', ':to'))
+                ->andWhere($qb->expr()->lte($this->getDatetimeFieldSql('t.end'), ':to'))
-                ->setParameter('to', $end, Type::DATETIME);
+                ->setParameter('to', $end);
         }
 
         if (null !== $user) {
@@ -285,15 +285,15 @@ class TimesheetRepository extends EntityRepository
         ;
 
         if (!empty($begin)) {
-            $qb->where($qb->expr()->gt('t.begin', ':from'));
+            $qb->andWhere($qb->expr()->gte($this->getDatetimeFieldSql('t.begin'), ':from'))
-            $qb->setParameter('from', $begin, Type::DATETIME);
+                ->setParameter('from', $begin);
         } else {
-            $qb->where($qb->expr()->isNotNull('t.begin'));
+            $qb->andWhere($qb->expr()->isNotNull('t.begin'));
         }
 
         if (!empty($end)) {
-            $qb->andWhere($qb->expr()->lt('t.end', ':to'))
+            $qb->andWhere($qb->expr()->lte($this->getDatetimeFieldSql('t.end'), ':to'))
-                ->setParameter('to', $end, Type::DATETIME);
+                ->setParameter('to', $end);
         } else {
             $qb->andWhere($qb->expr()->isNotNull('t.end'));
         }
@@ -674,7 +674,7 @@ class TimesheetRepository extends EntityRepository
         }
 
         if (null !== $query->getBegin()) {
-            $qb->andWhere('t.begin >= :begin')
+            $qb->andWhere($qb->expr()->gte($this->getDatetimeFieldSql('t.begin'), ':begin'))
                 ->setParameter('begin', $query->getBegin());
         }
 
@@ -685,7 +685,7 @@ class TimesheetRepository extends EntityRepository
         }
 
         if (null !== $query->getEnd()) {
-            $qb->andWhere('t.begin <= :end')
+            $qb->andWhere($qb->expr()->lte($this->getDatetimeFieldSql('t.begin'), ':end'))
                 ->setParameter('end', $query->getEnd());
         }
 
@@ -782,7 +782,7 @@ class TimesheetRepository extends EntityRepository
         }
 
         if (null !== $startFrom) {
-            $qb->andWhere($qb->expr()->gt('t.begin', ':begin'))
+            $qb->andWhere($qb->expr()->gte($this->getDatetimeFieldSql('t.begin'), ':begin'))
                 ->setParameter('begin', $startFrom);
         }
 
@@ -824,4 +824,9 @@ class TimesheetRepository extends EntityRepository
 
         $em->commit();
     }
+
+    private function getDatetimeFieldSql(string $field): string
+    {
+        return sprintf('CONVERT_TZ(%s, \'UTC\', t.timezone)', $field);
+    }
 }

templates/export/index.html.twig

@@ -30,6 +30,7 @@
         {% block box_title %}{{ 'export.filter'|trans }}{% endblock %}
         {% block box_before %}{{ form_start(form) }}{% endblock %}
         {% block box_body %}
+            {{ form_errors(form) }}
             {{ form_row(form.searchTerm) }}
             {{ form_row(form.daterange) }}
             {{ form_row(form.customer) }}

templates/invoice/index.html.twig

@@ -32,6 +32,7 @@
             {% block box_title %}{{ 'invoice.filter'|trans }}{% endblock %}
             {% block box_before %}{{ form_start(form) }}{% endblock %}
             {% block box_body %}
+                {{ form_errors(form) }}
                 {{ form_row(form.searchTerm) }}
                 {{ form_row(form.daterange) }}
                 {{ form_row(form.customer) }}