I really only wanted the day & month names to be translated according to the locale and wrote this twig extension. It accepts the normal DateTime->format() parameters and converts day & months names using strftime() if needed.
<?php
namespace AppBundle\Twig\Extension;
use Twig_Extension;
use Twig_SimpleFilter;
use DateTimeZone;
use DateTime;
class LocalizedDateExtension extends Twig_Extension
{
    protected static $conversionMap = [
        'D' => 'a',
        'l' => 'A',
        'M' => 'b',
        'F' => 'B',
    ];
    public function getFilters()
    {
        return [
            new Twig_SimpleFilter('localizeddate', [$this, 'localizeDate']),
        ];
    }
    protected static function createLocalizableTodo(&$formatString)
    {
        $newFormatString = '';
        $todo = [];
        $formatLength = mb_strlen($formatString);
        for ($i = 0; $i < $formatLength; $i++) {
            $char = $formatString[$i];
            if ('\'' === $char) {
                $newFormatString = $formatString[++$i]; //advance and add new character
            }
            if (array_key_exists($char, static::$conversionMap)) {
                $newFormatString.= '\!\L\O\C\A\L\I\Z\E\D\\'; //prefix char
                $todo[$char] = static::$conversionMap[$char];
            }
            $newFormatString.= $char;
        }
        $formatString = $newFormatString;
        return $todo;
    }
    public function localizeDate(DateTime $dateTime, $format, $timezone = null, $locale = null)
    {
        if (null !== $timezone && $dateTime->getTimezone()->getName() !== $timezone) {
            $dateTime = clone $dateTime;
            $dateTime->setTimezone(new DateTimeZone($timezone));
        }
        $todo = static::createLocalizableTodo($format);
        $output = $dateTime->format($format);
        //no localizeable parameters?
        if (0 === count($todo)) {
            return $output;
        }
        if ($locale !== null) {
            $currentLocale = setlocale(LC_TIME, '0');
            setlocale(LC_TIME, $locale);
        }
        if ($timezone !== null) {
            $currentTimezone = date_default_timezone_get();
            date_default_timezone_set($timezone);
        }
        //replace special parameters
        foreach ($todo as $placeholder => $parameter) {
            $output = str_replace('!LOCALIZED'.$placeholder, strftime('%'.$parameter, $dateTime->getTimestamp()), $output);
        }
        unset($parameter);
        if (isset($currentLocale)) {
            setlocale(LC_TIME, $currentLocale);
        }
        if (isset($currentTimezone)) {
            date_default_timezone_set($currentTimezone);
        }
        return $output;
    }
}