Merge pull request #354 from shikorism/feature/319-csv-import

チェックインデータのインポート

app/Ejaculation.php

@@ -47,6 +47,12 @@ class Ejaculation extends Model
         return $this->hasMany(Like::class);
     }
 
+    public function scopeOnlyWebCheckin(Builder $query)
+    {
+        return $query->where('ejaculations.source', null)
+            ->orWhere('ejaculations.source', '<>', Ejaculation::SOURCE_CSV);
+    }
+
     public function scopeWithLikes(Builder $query)
     {
         if (Auth::check()) {

app/Http/Controllers/HomeController.php

@@ -71,6 +71,7 @@ SQL
                 ->select('ejaculations.*')
                 ->with('user', 'tags')
                 ->withLikes()
+                ->onlyWebCheckin()
                 ->take(21)
                 ->get();
 

app/Http/Controllers/SettingController.php

@@ -3,7 +3,10 @@
 namespace App\Http\Controllers;
 
 use App\DeactivatedUser;
+use App\Ejaculation;
+use App\Exceptions\CsvImportException;
 use App\Services\CheckinCsvExporter;
+use App\Services\CheckinCsvImporter;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\DB;
@@ -72,6 +75,46 @@ class SettingController extends Controller
         return redirect()->route('setting.privacy')->with('status', 'プライバシー設定を更新しました。');
     }
 
+    public function import()
+    {
+        return view('setting.import');
+    }
+
+    public function storeImport(Request $request)
+    {
+        $validated = $request->validate([
+            'file' => 'required|file'
+        ], [], [
+            'file' => 'ファイル'
+        ]);
+
+        $file = $request->file('file');
+        if (!$file->isValid()) {
+            return redirect()->route('setting.import')->withErrors(['file' => 'ファイルのアップロードに失敗しました。']);
+        }
+
+        try {
+            set_time_limit(0);
+
+            $importer = new CheckinCsvImporter(Auth::user(), $file->path());
+            $imported = $importer->execute();
+
+            return redirect()->route('setting.import')->with('status', "{$imported}件のインポートに性交しました。");
+        } catch (CsvImportException $e) {
+            return redirect()->route('setting.import')->with('import_errors', $e->getErrors());
+        }
+    }
+
+    public function destroyImport()
+    {
+        Auth::user()
+            ->ejaculations()
+            ->where('ejaculations.source', Ejaculation::SOURCE_CSV)
+            ->delete();
+
+        return redirect()->route('setting.import')->with('status', '削除が完了しました。');
+    }
+
     public function export()
     {
         return view('setting.export');

app/Http/Controllers/TimelineController.php

@@ -19,6 +19,7 @@ class TimelineController extends Controller
             ->select('ejaculations.*')
             ->with('user', 'tags')
             ->withLikes()
+            ->onlyWebCheckin()
             ->paginate(21);
 
         return view('timeline.public')->with(compact('ejaculations'));

app/Http/Controllers/UserController.php

@@ -32,6 +32,7 @@ note,
 is_private,
 is_too_sensitive,
 link,
+source,
 to_char(lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC), 'YYYY/MM/DD HH24:MI') AS before_date,
 to_char(ejaculated_date - (lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)), 'FMDDD日 FMHH24時間 FMMI分') AS ejaculated_span
 SQL
@@ -154,6 +155,7 @@ note,
 is_private,
 is_too_sensitive,
 link,
+source,
 to_char(lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC), 'YYYY/MM/DD HH24:MI') AS before_date,
 to_char(ejaculated_date - (lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)), 'FMDDD日 FMHH24時間 FMMI分') AS ejaculated_span
 SQL

app/Services/CheckinCsvImporter.php

@@ -1,4 +1,5 @@
 <?php
+declare(strict_types=1);
 
 namespace App\Services;
 
@@ -8,12 +9,17 @@ use App\Rules\CsvDateTime;
 use App\Tag;
 use App\User;
 use Carbon\Carbon;
+use Illuminate\Database\QueryException;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Validator;
 use League\Csv\Reader;
+use Throwable;
 
 class CheckinCsvImporter
 {
+    /** @var int 取り込み件数の上限 */
+    private const IMPORT_LIMIT = 5000;
+
     /** @var User Target user */
     private $user;
     /** @var string CSV filename */
@@ -25,7 +31,11 @@ class CheckinCsvImporter
         $this->filename = $filename;
     }
 
-    public function execute()
+    /**
+     * インポート処理を実行します。
+     * @return int 取り込んだ件数
+     */
+    public function execute(): int
     {
         // Guess charset
         $charset = $this->guessCharset($this->filename);
@@ -38,7 +48,8 @@ class CheckinCsvImporter
         }
 
         // Import
-        DB::transaction(function () use ($csv) {
+        return DB::transaction(function () use ($csv) {
+            $alreadyImportedCount = $this->user->ejaculations()->where('ejaculations.source', Ejaculation::SOURCE_CSV)->count();
             $errors = [];
 
             if (!in_array('日時', $csv->getHeader(), true)) {
@@ -49,8 +60,15 @@ class CheckinCsvImporter
                 throw new CsvImportException(...$errors);
             }
 
+            $imported = 0;
             foreach ($csv->getRecords() as $offset => $record) {
                 $line = $offset + 1;
+                if (self::IMPORT_LIMIT <= $alreadyImportedCount + $imported) {
+                    $limit = self::IMPORT_LIMIT;
+                    $errors[] = "{$line} 行 : インポート機能で取り込めるデータは{$limit}件までに制限されています。これ以上取り込みできません。";
+                    throw new CsvImportException(...$errors);
+                }
+
                 $ejaculation = new Ejaculation(['user_id' => $this->user->id]);
 
                 $validator = Validator::make($record, [
@@ -78,15 +96,32 @@ class CheckinCsvImporter
                     continue;
                 }
 
-                $ejaculation->save();
+                DB::beginTransaction();
-                if (!empty($tags)) {
+                try {
-                    $ejaculation->tags()->sync(collect($tags)->pluck('id'));
+                    $ejaculation->save();
+                    if (!empty($tags)) {
+                        $ejaculation->tags()->sync(collect($tags)->pluck('id'));
+                    }
+                    DB::commit();
+                    $imported++;
+                } catch (QueryException $e) {
+                    DB::rollBack();
+                    if ($e->errorInfo[0] === '23505') {
+                        $errors[] = "{$line} 行 : すでにこの日時のチェックインデータが存在します。";
+                        continue;
+                    }
+                    throw $e;
+                } catch (Throwable $e) {
+                    DB::rollBack();
+                    throw $e;
                 }
             }
 
             if (!empty($errors)) {
                 throw new CsvImportException(...$errors);
             }
+
+            return $imported;
         });
     }
 
@@ -162,6 +197,9 @@ class CheckinCsvImporter
             if (strpos($tag, "\n") !== false) {
                 throw new CsvImportException("{$line} 行 : {$column}に改行を含めることはできません。");
             }
+            if (strpos($tag, ' ') !== false) {
+                throw new CsvImportException("{$line} 行 : {$column}にスペースを含めることはできません。");
+            }
 
             $tags[] = Tag::firstOrCreate(['name' => $tag]);
             if (count($tags) >= 32) {

app/Utilities/Formatter.php

@@ -92,4 +92,43 @@ class Formatter
 
         return implode(',', $srcset);
     }
+
+    /**
+     * php.ini書式のデータサイズを正規化します。
+     * @param mixed $val データサイズ
+     * @return string
+     */
+    public function normalizeIniBytes($val)
+    {
+        $val = trim($val);
+        $last = strtolower(substr($val, -1, 1));
+        if (ord($last) < 0x30 || ord($last) > 0x39) {
+            $bytes = substr($val, 0, -1);
+            switch ($last) {
+                case 'g':
+                    $bytes *= 1024;
+                    // fall through
+                    // no break
+                case 'm':
+                    $bytes *= 1024;
+                    // fall through
+                    // no break
+                case 'k':
+                    $bytes *= 1024;
+                    break;
+            }
+        } else {
+            $bytes = $val;
+        }
+
+        if ($bytes >= (1 << 30)) {
+            return ($bytes >> 30) . 'GB';
+        } elseif ($bytes >= (1 << 20)) {
+            return ($bytes >> 20) . 'MB';
+        } elseif ($bytes >= (1 << 10)) {
+            return ($bytes >> 10) . 'KB';
+        }
+
+        return $bytes . 'B';
+    }
 }

resources/assets/js/setting/import.js

@@ -0,0 +1,5 @@
+$('#destroy-form').on('submit', function () {
+    if (!confirm('本当に削除してもよろしいですか？')) {
+        return false;
+    }
+});

resources/views/components/ejaculation.blade.php

@@ -6,11 +6,14 @@
     </h5>
 </div>
 <!-- tags -->
-@if ($ejaculation->is_private || $ejaculation->tags->isNotEmpty())
+@if ($ejaculation->is_private || $ejaculation->source === 'csv' || $ejaculation->tags->isNotEmpty())
     <p class="mb-2">
         @if ($ejaculation->is_private)
             <span class="badge badge-warning"><span class="oi oi-lock-locked"></span> 非公開</span>
         @endif
+        @if ($ejaculation->source === 'csv')
+            <span class="badge badge-info"><span class="oi oi-cloud-upload"></span> インポート</span>
+        @endif
         @foreach ($ejaculation->tags as $tag)
             <a class="badge badge-secondary" href="{{ route('search', ['q' => $tag->name]) }}"><span class="oi oi-tag"></span> {{ $tag->name }}</a>
         @endforeach

resources/views/ejaculation/show.blade.php

@@ -34,11 +34,14 @@
                             <h5>{{ $ejaculatedSpan ?? '精通' }} <small class="text-muted">{{ $ejaculation->before_date }}{{ !empty($ejaculation->before_date) ? ' ～ ' : '' }}{{ $ejaculation->ejaculated_date->format('Y/m/d H:i') }}</small></h5>
                         </div>
                         <!-- tags -->
-                        @if ($ejaculation->is_private || $ejaculation->tags->isNotEmpty())
+                        @if ($ejaculation->is_private || $ejaculation->source === 'csv' || $ejaculation->tags->isNotEmpty())
                         <p class="mb-2">
                             @if ($ejaculation->is_private)
                                 <span class="badge badge-warning"><span class="oi oi-lock-locked"></span> 非公開</span>
                             @endif
+                            @if ($ejaculation->source === 'csv')
+                                <span class="badge badge-info"><span class="oi oi-cloud-upload"></span> インポート</span>
+                            @endif
                             @foreach ($ejaculation->tags as $tag)
                                 <a class="badge badge-secondary" href="{{ route('search', ['q' => $tag->name]) }}"><span class="oi oi-tag"></span> {{ $tag->name }}</a>
                             @endforeach

resources/views/setting/base.blade.php

@@ -10,6 +10,8 @@
                        href="{{ route('setting') }}"><span class="oi oi-person mr-1"></span> プロフィール</a>
                     <a class="list-group-item list-group-item-action {{ Route::currentRouteName() === 'setting.privacy' ? 'active' : '' }}"
                        href="{{ route('setting.privacy') }}"><span class="oi oi-shield mr-1"></span> プライバシー</a>
+                    <a class="list-group-item list-group-item-action {{ Route::currentRouteName() === 'setting.import' ? 'active' : '' }}"
+                       href="{{ route('setting.import') }}"><span class="oi oi-data-transfer-upload mr-1"></span> データのインポート</a>
                     <a class="list-group-item list-group-item-action {{ Route::currentRouteName() === 'setting.export' ? 'active' : '' }}"
                        href="{{ route('setting.export') }}"><span class="oi oi-data-transfer-download mr-1"></span> データのエクスポート</a>
                     <a class="list-group-item list-group-item-action {{ Route::currentRouteName() === 'setting.deactivate' ? 'active' : '' }}"

resources/views/setting/import.blade.php

@@ -0,0 +1,42 @@
+@extends('setting.base')
+
+@section('title', 'データのインポート')
+
+@section('tab-content')
+    <h3>データのインポート</h3>
+    <hr>
+    <p>外部で作成したチェックインデータをTissueに取り込むことができます。</p>
+    <form class="mt-4" action="{{ route('setting.import') }}" method="post" enctype="multipart/form-data">
+        {{ csrf_field() }}
+        <div class="form-group">
+            <strong>取り込むファイルを選択してください。</strong>
+            <small class="form-text text-muted">{{ Formatter::normalizeIniBytes(ini_get('upload_max_filesize')) }}までのCSVファイル、文字コードは Shift_JIS と UTF-8 (BOMなし) に対応しています。</small>
+            <input name="file" type="file" class="form-control-file {{ $errors->has('file') ? ' is-invalid' : '' }} mt-2">
+            @if ($errors->has('file'))
+                <div class="invalid-feedback">{{ $errors->first('file') }}</div>
+            @endif
+        </div>
+        @if (session('import_errors'))
+            <div class="alert alert-danger">
+                <p class="alert-heading"><span class="oi oi-warning"></span> <strong>インポートに失敗しました</strong></p>
+                @foreach (session('import_errors') as $err)
+                    <p class="mb-0">{{ $err }}</p>
+                @endforeach
+            </div>
+        @endif
+        <button type="submit" class="btn btn-primary mt-2">アップロード</button>
+    </form>
+    <h3 class="mt-5">インポートしたデータを一括削除</h3>
+    <hr>
+    <p class="mb-0">取り込んだチェックインデータをすべて削除することができます。データにミスがあってやり直したい場合などにお使いください。</p>
+    <p class="text-danger">ただし、インポート後に個別に手修正などしている場合、そのデータも失われてしまうことに注意してください！</p>
+    <form id="destroy-form" class="mt-4" action="{{ route('setting.import.destroy') }}" method="post">
+        {{ csrf_field() }}
+        {{ method_field('DELETE') }}
+        <button type="submit" class="btn btn-danger mt-2">データを削除</button>
+    </form>
+@endsection
+
+@push('script')
+    <script src="{{ mix('js/setting/import.js') }}"></script>
+@endpush

resources/views/user/profile.blade.php

@@ -40,11 +40,14 @@
                     <h5>{{ $ejaculation->ejaculated_span ?? '精通' }} <a href="{{ route('checkin.show', ['id' => $ejaculation->id]) }}" class="text-muted"><small>{{ $ejaculation->before_date }}{{ !empty($ejaculation->before_date) ? ' ～ ' : '' }}{{ $ejaculation->ejaculated_date->format('Y/m/d H:i') }}</small></a></h5>
                 </div>
                 <!-- tags -->
-                @if ($ejaculation->is_private || $ejaculation->tags->isNotEmpty())
+                @if ($ejaculation->is_private || $ejaculation->source === 'csv' || $ejaculation->tags->isNotEmpty())
                     <p class="mb-2">
                         @if ($ejaculation->is_private)
                             <span class="badge badge-warning"><span class="oi oi-lock-locked"></span> 非公開</span>
                         @endif
+                        @if ($ejaculation->source === 'csv')
+                            <span class="badge badge-info"><span class="oi oi-cloud-upload"></span> インポート</span>
+                        @endif
                         @foreach ($ejaculation->tags as $tag)
                             <a class="badge badge-secondary" href="{{ route('search', ['q' => $tag->name]) }}"><span class="oi oi-tag"></span> {{ $tag->name }}</a>
                         @endforeach

routes/web.php

@@ -36,6 +36,9 @@ Route::middleware('auth')->group(function () {
     Route::post('/setting/profile', 'SettingController@updateProfile')->name('setting.profile.update');
     Route::get('/setting/privacy', 'SettingController@privacy')->name('setting.privacy');
     Route::post('/setting/privacy', 'SettingController@updatePrivacy')->name('setting.privacy.update');
+    Route::get('/setting/import', 'SettingController@import')->name('setting.import');
+    Route::post('/setting/import', 'SettingController@storeImport')->name('setting.import');
+    Route::delete('/setting/import', 'SettingController@destroyImport')->name('setting.import.destroy');
     Route::get('/setting/export', 'SettingController@export')->name('setting.export');
     Route::get('/setting/export/csv', 'SettingController@exportToCsv')->name('setting.export.csv');
     Route::get('/setting/deactivate', 'SettingController@deactivate')->name('setting.deactivate');

tests/Unit/Services/CheckinCsvImporterTest.php

@@ -1,4 +1,5 @@
 <?php
+declare(strict_types=1);
 
 namespace Tests\Unit\Services;
 
@@ -240,6 +241,16 @@ class CheckinCsvImporterTest extends TestCase
         $importer->execute();
     }
 
+    public function testTagCantAcceptWhitespaceUTF8()
+    {
+        $user = factory(User::class)->create();
+        $this->expectException(CsvImportException::class);
+        $this->expectExceptionMessage('2 行 : タグ1にスペースを含めることはできません。');
+
+        $importer = new CheckinCsvImporter($user, __DIR__ . '/../../fixture/Csv/tag-whitespace.utf8.csv');
+        $importer->execute();
+    }
+
     public function testTagCanAccept32ColumnsUTF8()
     {
         $user = factory(User::class)->create();
@@ -265,4 +276,34 @@ class CheckinCsvImporterTest extends TestCase
         $this->assertSame(1, $user->ejaculations()->count());
         $this->assertEquals(Ejaculation::SOURCE_CSV, $ejaculation->source);
     }
+
+    public function testDontThrowUniqueKeyViolation()
+    {
+        $user = factory(User::class)->create();
+        factory(Ejaculation::class)->create([
+            'user_id' => $user->id,
+            'ejaculated_date' => Carbon::create(2020, 1, 23, 6, 1, 0, 'Asia/Tokyo')
+        ]);
+
+        $this->expectException(CsvImportException::class);
+        $this->expectExceptionMessage('2 行 : すでにこの日時のチェックインデータが存在します。');
+
+        $importer = new CheckinCsvImporter($user, __DIR__ . '/../../fixture/Csv/date.utf8.csv');
+        $importer->execute();
+    }
+
+    public function testRecordLimit()
+    {
+        $user = factory(User::class)->create();
+        factory(Ejaculation::class, 5000)->create([
+            'user_id' => $user->id,
+            'source' => Ejaculation::SOURCE_CSV
+        ]);
+
+        $this->expectException(CsvImportException::class);
+        $this->expectExceptionMessage('2 行 : インポート機能で取り込めるデータは5000件までに制限されています。これ以上取り込みできません。');
+
+        $importer = new CheckinCsvImporter($user, __DIR__ . '/../../fixture/Csv/link.utf8.csv');
+        $importer->execute();
+    }
 }

tests/fixture/Csv/tag-whitespace.utf8.csv

@@ -0,0 +1,2 @@
+日時,タグ1
+2020/01/23 06:01,"空白を含む タグ"

webpack.mix.js

@@ -16,6 +16,7 @@ mix.js('resources/assets/js/app.js', 'public/js')
     .js('resources/assets/js/home.js', 'public/js')
     .js('resources/assets/js/user/stats.js', 'public/js/user')
     .js('resources/assets/js/setting/privacy.js', 'public/js/setting')
+    .js('resources/assets/js/setting/import.js', 'public/js/setting')
     .js('resources/assets/js/setting/deactivate.js', 'public/js/setting')
     .ts('resources/assets/js/checkin.ts', 'public/js')
     .sass('resources/assets/sass/app.scss', 'public/css')