Rewrote Wallabag v1 import

app/config/config.yml

@@ -33,6 +33,7 @@ wallabag_core:
 
 wallabag_import:
     allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain']
+    resource_dir: "%kernel.root_dir%/../web/uploads/import"
 
 # Twig Configuration
 twig:

src/Wallabag/ImportBundle/Command/ImportCommand.php

@@ -7,118 +7,47 @@ use Symfony\Component\Config\Definition\Exception\Exception;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Console\Helper\ProgressBar;
-use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\CoreBundle\Tools\Utils;
 
 class ImportCommand extends ContainerAwareCommand
 {
     protected function configure()
     {
         $this
-            ->setName('wallabag:import')
-            ->setDescription('Import entries from JSON file')
-            ->addArgument(
-                'userId',
-                InputArgument::REQUIRED,
-                'user ID to populate'
-            );
+            ->setName('wallabag:import-v1')
+            ->setDescription('Import entries from a JSON export from a wallabag v1 instance')
+            ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate')
+            ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file')
+        ;
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $now = new \DateTime();
-        $output->writeln('<comment>Start : '.$now->format('d-m-Y G:i:s').' ---</comment>');
-
-        // Importing CSV on DB via Doctrine ORM
-        $this->import($input, $output);
-
-        $now = new \DateTime();
-        $output->writeln('<comment>End : '.$now->format('d-m-Y G:i:s').' ---</comment>');
-    }
-
-    protected function import(InputInterface $input, OutputInterface $output)
-    {
-        // Getting php array of data from CSV
-        $data = $this->get($input, $output);
+        $output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---');
 
         $em = $this->getContainer()->get('doctrine')->getManager();
         // Turning off doctrine default logs queries for saving memory
         $em->getConnection()->getConfiguration()->setSQLLogger(null);
 
-        // Define the size of record, the frequency for persisting the data and the current index of records
-        $size = count($data);
-        $batchSize = 20;
-        $i = 1;
-
-        $user = $em->getRepository('WallabagUserBundle:User')
-            ->findOneById($input->getArgument('userId'));
+        $user = $em->getRepository('WallabagUserBundle:User')->findOneById($input->getArgument('userId'));
 
         if (!is_object($user)) {
-            throw new Exception('User not found');
+            throw new Exception(sprintf('User with id "%s" not found', $input->getArgument('userId')));
         }
 
-        $progress = new ProgressBar($output, $size);
-        $progress->start();
+        $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import');
+        $res = $wallabag
+            ->setUser($user)
+            ->setFilepath($input->getArgument('filepath'))
+            ->import();
 
-        foreach ($data as $object) {
-            $array = (array) $object;
-            $entry = $em->getRepository('WallabagCoreBundle:Entry')
-                ->findOneByUrl($array['url']);
-
-            if (!is_object($entry)) {
-                $entry = new Entry($user);
-                $entry->setUrl($array['url']);
-            }
-
-            $entry->setTitle($array['title']);
-            $entry->setArchived($array['is_read']);
-            $entry->setStarred($array['is_fav']);
-            $entry->setContent($array['content']);
-            $entry->setReadingTime(Utils::getReadingTime($array['content']));
-
-            $em->persist($entry);
-
-            if (($i % $batchSize) === 0) {
-                $em->flush();
-                $progress->advance($batchSize);
-
-                $now = new \DateTime();
-                $output->writeln(' of entries imported ... | '.$now->format('d-m-Y G:i:s'));
-            }
-            ++$i;
+        if (true === $res) {
+            $summary = $wallabag->getSummary();
+            $output->writeln('<info>'.$summary['imported'].' imported</info>');
+            $output->writeln('<comment>'.$summary['skipped'].' already saved</comment>');
         }
 
-        $em->flush();
         $em->clear();
-        $progress->finish();
-    }
 
-    protected function convert($filename)
-    {
-        if (!file_exists($filename) || !is_readable($filename)) {
-            return false;
-        }
-
-        $header = null;
-        $data = array();
-
-        if (($handle = fopen($filename, 'r')) !== false) {
-            while (($row = fgets($handle)) !== false) {
-                $data = json_decode($row);
-            }
-            fclose($handle);
-        }
-
-        return $data;
-    }
-
-    protected function get(InputInterface $input, OutputInterface $output)
-    {
-        $filename = __DIR__.'/../../../../web/uploads/import/'.$input->getArgument('userId').'.json';
-
-        $data = $this->convert($filename);
-
-        return $data;
+        $output->writeln('End : '.(new \DateTime())->format('d-m-Y G:i:s').' ---');
     }
 }

src/Wallabag/ImportBundle/Controller/PocketController.php

@@ -51,7 +51,7 @@ class PocketController extends Controller
 
         if (true === $pocket->import()) {
             $summary = $pocket->getSummary();
-            $message = $summary['imported'].' entrie(s) imported, '.$summary['skipped'].' already saved.';
+            $message = 'Import summary: '.$summary['imported'].' imported, '.$summary['skipped'].' already saved.';
         }
 
         $this->get('session')->getFlashBag()->add(

src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Symfony\Bundle\FrameworkBundle\Controller\Controller;
+use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
+use Symfony\Component\HttpFoundation\Request;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+
+class WallabagV1Controller extends Controller
+{
+    /**
+     * @Route("/import/wallabag-v1", name="import_wallabag_v1")
+     */
+    public function indexAction(Request $request)
+    {
+        $importForm = $this->createForm(new UploadImportType());
+        $importForm->handleRequest($request);
+        $user = $this->getUser();
+
+        if ($importForm->isValid()) {
+            $file = $importForm->get('file')->getData();
+            $name = $user->getId().'.json';
+
+            if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+                $wallabag = $this->get('wallabag_import.wallabag_v1.import');
+                $res = $wallabag
+                    ->setUser($this->getUser())
+                    ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
+                    ->import();
+
+                $message = 'Import failed, please try again.';
+                if (true === $res) {
+                    $summary = $wallabag->getSummary();
+                    $message = 'Import summary: '.$summary['imported'].' imported, '.$summary['skipped'].' already saved.';
+
+                    @unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
+                }
+
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    $message
+                );
+
+                return $this->redirect($this->generateUrl('homepage'));
+            } else {
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    'Error while processing import. Please verify your import file.'
+                );
+            }
+        }
+
+        return $this->render('WallabagImportBundle:WallabagV1:index.html.twig', [
+            'form' => $importForm->createView(),
+        ]);
+    }
+}

src/Wallabag/ImportBundle/DependencyInjection/Configuration.php

@@ -17,6 +17,8 @@ class Configuration implements ConfigurationInterface
                 ->arrayNode('allow_mimetypes')
                     ->prototype('scalar')->end()
                 ->end()
+                ->scalarNode('resource_dir')
+                ->end()
             ->end()
         ;
 

src/Wallabag/ImportBundle/DependencyInjection/WallabagImportExtension.php

@@ -14,6 +14,7 @@ class WallabagImportExtension extends Extension
         $configuration = new Configuration();
         $config = $this->processConfiguration($configuration, $configs);
         $container->setParameter('wallabag_import.allow_mimetypes', $config['allow_mimetypes']);
+        $container->setParameter('wallabag_import.resource_dir', $config['resource_dir']);
 
         $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
         $loader->load('services.yml');

src/Wallabag/ImportBundle/Import/PocketImport.php

@@ -141,7 +141,7 @@ class PocketImport implements ImportInterface
 
         $entries = $response->json();
 
-        $this->parsePocketEntries($entries['list']);
+        $this->parseEntries($entries['list']);
 
         return true;
     }
@@ -194,11 +194,9 @@ class PocketImport implements ImportInterface
      *
      * @param $entries
      */
-    private function parsePocketEntries($entries)
+    private function parseEntries($entries)
     {
         foreach ($entries as $pocketEntry) {
-            $entry = new Entry($this->user);
-
             $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
 
             $existingEntry = $this->em
@@ -210,6 +208,7 @@ class PocketImport implements ImportInterface
                 continue;
             }
 
+            $entry = new Entry($this->user);
             $entry = $this->contentProxy->updateEntry($entry, $url);
 
             // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted

src/Wallabag/ImportBundle/Import/WallabagV1Import.php

@@ -0,0 +1,137 @@
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+use Doctrine\ORM\EntityManager;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\CoreBundle\Tools\Utils;
+
+class WallabagV1Import implements ImportInterface
+{
+    private $user;
+    private $em;
+    private $logger;
+    private $skippedEntries = 0;
+    private $importedEntries = 0;
+    private $filepath;
+
+    public function __construct(EntityManager $em)
+    {
+        $this->em = $em;
+        $this->logger = new NullLogger();
+    }
+
+    public function setLogger(LoggerInterface $logger)
+    {
+        $this->logger = $logger;
+    }
+
+    /**
+     * We define the user in a custom call because on the import command there is no logged in user.
+     * So we can't retrieve user from the `security.token_storage` service.
+     *
+     * @param User $user
+     */
+    public function setUser(User $user)
+    {
+        $this->user = $user;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getName()
+    {
+        return 'Wallabag v1';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDescription()
+    {
+        return 'This importer will import all your wallabag v1 articles.';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function import()
+    {
+        if (!$this->user) {
+            $this->logger->error('WallabagV1Import: user is not defined');
+
+            return false;
+        }
+
+        if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+            $this->logger->error('WallabagV1Import: unable to read file', array('filepath' => $this->filepath));
+
+            return false;
+        }
+
+        $this->parseEntries(json_decode(file_get_contents($this->filepath), true));
+
+        return true;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSummary()
+    {
+        return [
+            'skipped' => $this->skippedEntries,
+            'imported' => $this->importedEntries,
+        ];
+    }
+
+    /**
+     * Set file path to the json file.
+     *
+     * @param string $filepath
+     */
+    public function setFilepath($filepath)
+    {
+        $this->filepath = $filepath;
+
+        return $this;
+    }
+
+    /**
+     * @param $entries
+     */
+    private function parseEntries($entries)
+    {
+        foreach ($entries as $importedEntry) {
+            $existingEntry = $this->em
+                ->getRepository('WallabagCoreBundle:Entry')
+                ->existByUrlAndUserId($importedEntry['url'], $this->user->getId());
+
+            if (false !== $existingEntry) {
+                ++$this->skippedEntries;
+                continue;
+            }
+
+            // @see ContentProxy->updateEntry
+            $entry = new Entry($this->user);
+            $entry->setUrl($importedEntry['url']);
+            $entry->setTitle($importedEntry['title']);
+            $entry->setArchived($importedEntry['is_read']);
+            $entry->setStarred($importedEntry['is_fav']);
+            $entry->setContent($importedEntry['content']);
+            $entry->setReadingTime(Utils::getReadingTime($importedEntry['content']));
+            $entry->setDomainName(parse_url($importedEntry['url'], PHP_URL_HOST));
+
+            $this->em->persist($entry);
+            ++$this->importedEntries;
+        }
+
+        $this->em->flush();
+    }
+}

src/Wallabag/ImportBundle/Resources/config/services.yml

@@ -18,3 +18,10 @@ services:
         calls:
             - [ setClient, [ "@wallabag_import.pocket.client" ] ]
             - [ setLogger, [ "@logger" ]]
+
+    wallabag_import.wallabag_v1.import:
+        class: Wallabag\ImportBundle\Import\WallabagV1Import
+        arguments:
+            - "@doctrine.orm.entity_manager"
+        calls:
+            - [ setLogger, [ "@logger" ]]

src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig

@@ -9,6 +9,7 @@
             {% trans %}Welcome on wallabag importer. Please select your previous service that you want to migrate.{% endtrans %}
             <ul>
                 <li><a href="{{ path('import_pocket') }}">Pocket</a></li>
+                <li><a href="{{ path('import_wallabag_v1') }}">Wallabag v1</a></li>
             </ul>
         </div>
     </div>

src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig

@@ -0,0 +1,30 @@
+{% extends "WallabagCoreBundle::layout.html.twig" %}
+{% block title %}{% trans %}import{% endtrans %}{% endblock %}
+
+{% block content %}
+
+<div class="row">
+    <div class="col s12">
+        <div class="card-panel settings">
+            <div class="row">
+                <div class="col s12">
+                    {{ form_start(form, {'method': 'POST'}) }}
+                        {{ form_errors(form) }}
+                        <div class="row">
+                            <div class="input-field col s12">
+                                <p>{% trans %}Please select your wallabag export and click on the below button to upload and import it.{% endtrans %}</p>
+                                {{ form_errors(form.file) }}
+                                {{ form_widget(form.file) }}
+                            </div>
+                        </div>
+                        <div class="hidden">{{ form_rest(form) }}</div>
+                        <button class="btn waves-effect waves-light" type="submit" name="action">
+                            {% trans %}Upload file{% endtrans %}
+                        </button>
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}

src/Wallabag/ImportBundle/Tests/Import/WallabagV1ImportTest.php

@@ -0,0 +1,96 @@
+<?php
+
+namespace Wallabag\ImportBundle\Tests\Import;
+
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\ImportBundle\Import\WallabagV1Import;
+use Monolog\Logger;
+use Monolog\Handler\TestHandler;
+
+class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
+{
+    protected $user;
+    protected $em;
+    protected $logHandler;
+
+    private function getWallabagV1Import($unsetUser = false)
+    {
+        $this->user = new User();
+
+        $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $pocket = new WallabagV1Import($this->em);
+
+        $this->logHandler = new TestHandler();
+        $logger = new Logger('test', array($this->logHandler));
+        $pocket->setLogger($logger);
+
+        if (false === $unsetUser) {
+            $pocket->setUser($this->user);
+        }
+
+        return $pocket;
+    }
+
+    public function testInit()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import();
+
+        $this->assertEquals('Wallabag v1', $wallabagV1Import->getName());
+        $this->assertEquals('This importer will import all your wallabag v1 articles.', $wallabagV1Import->getDescription());
+    }
+
+    public function testImport()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
+
+        $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $entryRepo->expects($this->exactly(3))
+            ->method('existByUrlAndUserId')
+            ->will($this->onConsecutiveCalls(false, true, false));
+
+        $this->em
+            ->expects($this->any())
+            ->method('getRepository')
+            ->willReturn($entryRepo);
+
+        $res = $wallabagV1Import->import();
+
+        $this->assertTrue($res);
+        $this->assertEquals(['skipped' => 1, 'imported' => 2], $wallabagV1Import->getSummary());
+    }
+
+    public function testImportBadFile()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import();
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx');
+
+        $res = $wallabagV1Import->import();
+
+        $this->assertFalse($res);
+
+        $records = $this->logHandler->getRecords();
+        $this->assertContains('WallabagV1Import: unable to read file', $records[0]['message']);
+        $this->assertEquals('ERROR', $records[0]['level_name']);
+    }
+
+    public function testImportUserNotDefined()
+    {
+        $wallabagV1Import = $this->getWallabagV1Import(true);
+        $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
+
+        $res = $wallabagV1Import->import();
+
+        $this->assertFalse($res);
+
+        $records = $this->logHandler->getRecords();
+        $this->assertContains('WallabagV1Import: user is not defined', $records[0]['message']);
+        $this->assertEquals('ERROR', $records[0]['level_name']);
+    }
+}