lint

Fix Cien resolver

.circleci/config.yml

@@ -24,8 +24,13 @@ commands:
       - checkout
       - run: sudo apt update
       - run: sudo apt install -y libpq-dev
+      - run: sudo pecl install -f xdebug-2.9.8 && sudo docker-php-ext-enable xdebug
       - run: sudo docker-php-ext-install zip
       - run: sudo docker-php-ext-install pdo_pgsql
+      - run:
+          command: |
+            curl -sSL "https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-x64.tar.xz" | sudo tar --strip-components=2 -xJ -C /usr/local/bin/ node-v12.16.3-linux-x64/bin/node
+            curl https://www.npmjs.com/install.sh | sudo bash
   restore_composer:
     steps:
       - restore_cache:
@@ -101,6 +106,12 @@ jobs:
           command: yarn run stylelint
           when: always
 
+      # Run eslint
+      - run:
+          name: eslint
+          command: yarn run eslint
+          when: always
+
       # Run unit test
       - run:
           command: |

.editorconfig

@@ -11,7 +11,7 @@ insert_final_newline = true
 [*.md]
 trim_trailing_whitespace = false
 
-[*.yml]
+[*.{yml,yaml}]
 indent_size = 2
 
 [*.json]

.env.example

@@ -2,12 +2,16 @@ APP_NAME=Tissue
 APP_ENV=local
 APP_KEY=
 APP_DEBUG=true
-APP_LOG_LEVEL=debug
 APP_URL=http://localhost
 
+LOG_CHANNEL=stack
+
 # テストにモックを使用するか falseの場合は実際のHTML等を取得してテストする
 TEST_USE_HTTP_MOCK=true
 
+# テスト用のスナップショットを更新する場合はtrueにする (TEST_USE_HTTP_MOCKと重複させないよう注意)
+TEST_UPDATE_SNAPSHOT=false
+
 DB_CONNECTION=pgsql
 DB_HOST=db
 DB_PORT=5432
@@ -33,13 +37,23 @@ MAIL_ENCRYPTION=tls
 MAIL_FROM_ADDRESS=support@mail.shikorism.net
 MAIL_FROM_NAME=Tissue
 
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=us-east-1
+AWS_BUCKET=
+
 SPARKPOST_SECRET=
 
 PUSHER_APP_ID=
 PUSHER_APP_KEY=
 PUSHER_APP_SECRET=
 
+MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
+MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
+
 # (Optional) reCAPTCHA Key
 # https://www.google.com/recaptcha
 NOCAPTCHA_SECRET=
 NOCAPTCHA_SITEKEY=
+
+SENTRY_LARAVEL_DSN=

.eslintrc.js

@@ -0,0 +1,38 @@
+module.exports = {
+    env: {
+        browser: true,
+        es2020: true,
+        node: true,
+    },
+    extends: [
+        'eslint:recommended',
+        'plugin:react/recommended',
+        'plugin:prettier/recommended',
+        'plugin:@typescript-eslint/recommended',
+        'prettier',
+        'prettier/@typescript-eslint',
+        'prettier/react',
+    ],
+    parser: '@typescript-eslint/parser',
+    parserOptions: {
+        ecmaVersion: 11,
+        ecmaFeatures: {
+            jsx: true,
+        },
+        sourceType: 'module',
+    },
+    plugins: ['prettier', 'react', '@typescript-eslint', 'jquery'],
+    rules: {
+        '@typescript-eslint/explicit-module-boundary-types': 0,
+        '@typescript-eslint/no-explicit-any': 0,
+        '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }],
+        'jquery/no-ajax': 2,
+        'jquery/no-ajax-events': 2,
+        'react/prop-types': 0,
+    },
+    settings: {
+        react: {
+            version: 'detect',
+        },
+    },
+};

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+
+  # Maintain dependencies for npm
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  # Maintain dependencies for Composer
+  - package-ecosystem: "composer"
+    directory: "/"
+    schedule:
+      interval: "daily"

.gitignore

@@ -5,6 +5,8 @@
 /public/hot
 /public/storage
 /public/mix-manifest.json
+/public/report.html
+/public/apidoc.html
 /storage/*.key
 /vendor
 /.idea
@@ -15,6 +17,8 @@ Homestead.yaml
 npm-debug.log
 yarn-error.log
 .env
+.env.backup
+.phpunit.result.cache
 *.iml
 .php_cs
 .php_cs.cache

.prettierrc

@@ -0,0 +1,5 @@
+{
+  "arrowParens": "always",
+  "singleQuote": true,
+  "printWidth": 120
+}

Dockerfile

@@ -5,12 +5,11 @@ FROM php:7.3-apache
 ENV APACHE_DOCUMENT_ROOT /var/www/html/public
 
 RUN apt-get update \
-    && apt-get install -y git libpq-dev unzip \
-    && docker-php-ext-install pdo_pgsql \
+    && apt-get install -y git libpq-dev unzip libicu-dev \
+    && docker-php-ext-install pdo_pgsql intl \
     && pecl install xdebug \
     && curl -sS https://getcomposer.org/installer | php \
     && mv composer.phar /usr/local/bin/composer \
-    && composer global require hirak/prestissimo \
     && sed -ri -e 's!/var/www/html!${APACHE_DOCUMENT_ROOT}!g' /etc/apache2/sites-available/*.conf \
     && sed -ri -e 's!/var/www/!${APACHE_DOCUMENT_ROOT}!g' /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf \
     && a2enmod rewrite

Procfile

@@ -0,0 +1 @@
+web: vendor/bin/heroku-php-apache2 public/

README.md

@@ -7,12 +7,12 @@ a.k.a. shikorism.net
 
 ## 構成
 
-- Laravel 5.5
-- Bootstrap 4.3.1
+- Laravel 6
+- Bootstrap 4.4.1
 
 ## 実行環境
 
-- PHP 7.1
+- PHP 7.3
 - PostgreSQL 9.6
 
 ## 開発環境の構築

app/CheckinWebhook.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Support\Str;
+
+class CheckinWebhook extends Model
+{
+    use SoftDeletes;
+
+    /** @var int ユーザーごとの作成数制限 */
+    const PER_USER_LIMIT = 10;
+
+    public $incrementing = false;
+    protected $keyType = 'string';
+
+    protected $fillable = ['name'];
+
+    protected static function boot()
+    {
+        parent::boot();
+
+        self::creating(function (CheckinWebhook $webhook) {
+            $webhook->id = Str::random(64);
+        });
+    }
+
+    public function user()
+    {
+        return $this->belongsTo(User::class);
+    }
+
+    public function isAvailable()
+    {
+        return $this->user !== null;
+    }
+}

app/Console/Commands/DedupTags.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Tag;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class DedupTags extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'tissue:tag:dedup {--dry-run}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Deduplicate tags';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        if ($this->option('dry-run')) {
+            $this->warn('dry-runモードで実行します。');
+        } else {
+            if (!$this->confirm('dry-runオプションが付いてないけど、本当に実行しますか？')) {
+                return;
+            }
+        }
+
+        DB::transaction(function () {
+            $duplicatedTags = DB::table('tags')
+                ->select('name', DB::raw('count(*)'))
+                ->groupBy('name')
+                ->having(DB::raw('count(*)'), '>=', 2)
+                ->get();
+
+            $this->info($duplicatedTags->count() . ' duplicated tags found.');
+
+            foreach ($duplicatedTags as $tag) {
+                $this->line('Tag name: ' . $tag->name);
+
+                $tagIds = Tag::where('name', $tag->name)->orderBy('id')->pluck('id');
+                $newId = $tagIds->first();
+                $dropIds = $tagIds->slice(1);
+
+                $this->line('  New ID: ' . $newId);
+                $this->line('  Drop IDs: ' . $dropIds->implode(', '));
+
+                if ($this->option('dry-run')) {
+                    continue;
+                }
+
+                // 同じタグ名でIDが違うものについて、全て統一する
+                foreach (['ejaculation_tag', 'metadata_tag'] as $table) {
+                    DB::table($table)
+                        ->whereIn('tag_id', $dropIds)
+                        ->update(['tag_id' => $newId]);
+                }
+                DB::table('tags')->whereIn('id', $dropIds)->delete();
+
+                // 統一した上で、重複しているレコードを削除する
+                DB::delete(
+                    <<<SQL
+DELETE FROM ejaculation_tag
+WHERE id IN (
+    SELECT id
+    FROM (
+        SELECT id, row_number() OVER (PARTITION BY ejaculation_id, tag_id ORDER BY id) AS ord
+        FROM ejaculation_tag
+    ) t
+    WHERE ord > 1
+)
+SQL
+                );
+                DB::delete(
+                    <<<SQL
+DELETE FROM metadata_tag
+WHERE id IN (
+    SELECT id
+    FROM (
+        SELECT id, row_number() OVER (PARTITION BY metadata_url, tag_id ORDER BY id) AS ord
+        FROM metadata_tag
+    ) t
+    WHERE ord > 1
+)
+SQL
+                );
+            }
+        });
+
+        $this->info('Done!');
+    }
+}

app/Console/Commands/NormalizeTags.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Tag;
+use App\Utilities\Formatter;
+use DB;
+use Illuminate\Console\Command;
+
+class NormalizeTags extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'tissue:tag:normalize';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Normalize tags';
+
+    private $formatter;
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct(Formatter $formatter)
+    {
+        parent::__construct();
+        $this->formatter = $formatter;
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $start = hrtime(true);
+
+        DB::transaction(function () {
+            /** @var Tag $tag */
+            foreach (Tag::query()->cursor() as $tag) {
+                $normalizedName = $this->formatter->normalizeTagName($tag->name);
+                $this->line("{$tag->name} : {$normalizedName}");
+                $tag->normalized_name = $normalizedName;
+                $tag->save();
+            }
+        });
+
+        $elapsed = (hrtime(true) - $start) / 1e+9;
+        $this->info("Done! ({$elapsed} sec)");
+    }
+}

app/ContentProvider.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class ContentProvider extends Model
+{
+    public $incrementing = false;
+    protected $primaryKey = 'host';
+    protected $keyType = 'string';
+
+    protected $fillable = [
+        'host',
+        'robots',
+        'robots_cached_at',
+    ];
+
+    protected $dates = [
+        'created_at',
+        'updated_at',
+        'robots_cached_at',
+    ];
+}

app/DeactivatedUser.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+/**
+ * 削除済Userのユーザー名履歴
+ */
+class DeactivatedUser extends Model
+{
+    public $incrementing = false;
+    protected $keyType = 'string';
+
+    protected $fillable = [
+        'name'
+    ];
+}

app/Ejaculation.php

@@ -12,10 +12,15 @@ class Ejaculation extends Model
 {
     use HasEagerLimit;
 
+    const SOURCE_WEB = 'web';
+    const SOURCE_CSV = 'csv';
+    const SOURCE_WEBHOOK = 'webhook';
+
     protected $fillable = [
         'user_id', 'ejaculated_date',
-        'note', 'geo_latitude', 'geo_longitude', 'link',
-        'is_private'
+        'note', 'geo_latitude', 'geo_longitude', 'link', 'source',
+        'is_private', 'is_too_sensitive', 'discard_elapsed_time',
+        'checkin_webhook_id'
     ];
 
     protected $dates = [
@@ -44,6 +49,11 @@ class Ejaculation extends Model
         return $this->hasMany(Like::class);
     }
 
+    public function scopeVisibleToTimeline(Builder $query)
+    {
+        return $query->whereIn('ejaculations.source', [Ejaculation::SOURCE_WEB, Ejaculation::SOURCE_WEBHOOK]);
+    }
+
     public function scopeWithLikes(Builder $query)
     {
         if (Auth::check()) {
@@ -56,6 +66,7 @@ class Ejaculation extends Model
                     },
                     'likes.user' => function ($query) {
                         $query->where('is_protected', false)
+                            ->where('private_likes', false)
                             ->orWhere('id', Auth::id());
                     }
                 ])
@@ -72,11 +83,55 @@ class Ejaculation extends Model
                         $query->latest()->take(10);
                     },
                     'likes.user' => function ($query) {
-                        $query->where('is_protected', false);
+                        $query->where('is_protected', false)
+                            ->where('private_likes', false);
                     }
                 ])
                 ->withCount('likes')
                 ->addSelect(DB::raw('0 as is_liked'));
         }
     }
+
+    /**
+     * このチェックインと同じ情報を流用してチェックインするためのURLを生成
+     * @return string
+     */
+    public function makeCheckinURL(): string
+    {
+        return route('checkin', [
+            'link' => $this->link,
+            'tags' => $this->textTags(),
+            'is_private' => $this->is_private,
+            'is_too_sensitive' => $this->is_too_sensitive,
+        ]);
+    }
+
+    public function ejaculatedSpan(): string
+    {
+        if (array_key_exists('ejaculated_span', $this->attributes)) {
+            if ($this->ejaculated_span === null) {
+                return '精通';
+            }
+            if ($this->discard_elapsed_time) {
+                return '0日 0時間 0分'; // TODO: 気の効いたフレーズにする
+            }
+
+            return $this->ejaculated_span;
+        } else {
+            $previous = Ejaculation::select('ejaculated_date')
+                ->where('user_id', $this->user_id)
+                ->where('ejaculated_date', '<', $this->ejaculated_date)
+                ->orderByDesc('ejaculated_date')
+                ->first();
+
+            if ($previous === null) {
+                return '精通';
+            }
+            if ($this->discard_elapsed_time) {
+                return '0日 0時間 0分';
+            }
+
+            return $this->ejaculated_date->diff($previous->ejaculated_date)->format('%a日 %h時間 %i分');
+        }
+    }
 }

app/Exceptions/CsvImportException.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Exceptions;
+
+use Illuminate\Support\Arr;
+use Throwable;
+
+class CsvImportException extends \RuntimeException
+{
+    /** @var string[] */
+    private $errors;
+
+    /**
+     * CsvImportException constructor.
+     * @param string[] $errors
+     */
+    public function __construct(...$errors)
+    {
+        parent::__construct(Arr::first($errors));
+        $this->errors = $errors;
+    }
+
+    /**
+     * @return string[]
+     */
+    public function getErrors(): array
+    {
+        return $this->errors;
+    }
+}

app/Exceptions/Handler.php

@@ -4,7 +4,10 @@ namespace App\Exceptions;
 
 use Exception;
 use Illuminate\Auth\AuthenticationException;
+use Illuminate\Database\Eloquent\ModelNotFoundException;
 use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
+use Illuminate\Http\JsonResponse;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
 
 class Handler extends ExceptionHandler
 {
@@ -20,18 +23,23 @@ class Handler extends ExceptionHandler
         \Illuminate\Database\Eloquent\ModelNotFoundException::class,
         \Illuminate\Session\TokenMismatchException::class,
         \Illuminate\Validation\ValidationException::class,
+        \App\MetadataResolver\ResolverCircuitBreakException::class,
     ];
 
     /**
      * Report or log an exception.
      *
-     * This is a great spot to send exceptions to Sentry, Bugsnag, etc.
-     *
      * @param  \Exception  $exception
      * @return void
+     *
+     * @throws \Exception
      */
     public function report(Exception $exception)
     {
+        if (app()->bound('sentry') && $this->shouldReport($exception)) {
+            app('sentry')->captureException($exception);
+        }
+
         parent::report($exception);
     }
 
@@ -40,7 +48,9 @@ class Handler extends ExceptionHandler
      *
      * @param  \Illuminate\Http\Request  $request
      * @param  \Exception  $exception
-     * @return \Illuminate\Http\Response
+     * @return \Symfony\Component\HttpFoundation\Response
+     *
+     * @throws \Exception
      */
     public function render($request, Exception $exception)
     {
@@ -62,4 +72,28 @@ class Handler extends ExceptionHandler
 
         return redirect()->guest(route('login'));
     }
+
+    protected function prepareException(Exception $e)
+    {
+        if (!config('app.debug') && $e instanceof ModelNotFoundException) {
+            return new NotFoundHttpException('Resource not found.', $e);
+        }
+
+        return parent::prepareException($e);
+    }
+
+    protected function prepareJsonResponse($request, Exception $e)
+    {
+        $status = $this->isHttpException($e) ? $e->getStatusCode() : 500;
+
+        return new JsonResponse(
+            [
+                'status' => $status,
+                'error' => $this->convertExceptionToArray($e),
+            ],
+            $status,
+            $this->isHttpException($e) ? $e->getHeaders() : [],
+            JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES
+        );
+    }
 }

app/Http/Controllers/Admin/InfoController.php

@@ -44,12 +44,10 @@ class InfoController extends Controller
         return redirect()->route('admin.info.edit', ['info' => $info])->with('status', 'お知らせを更新しました。');
     }
 
-    public function edit($id)
+    public function edit(Information $info)
     {
-        $information = Information::findOrFail($id);
-
         return view('admin.info.edit')->with([
-            'info' => $information,
+            'info' => $info,
             'categories' => Information::CATEGORIES
         ]);
     }

app/Http/Controllers/Api/CardController.php

@@ -2,55 +2,23 @@
 
 namespace App\Http\Controllers\Api;
 
-use App\Metadata;
-use App\MetadataResolver\MetadataResolver;
-use App\Tag;
-use App\Utilities\Formatter;
+use App\MetadataResolver\DeniedHostException;
+use App\Services\MetadataResolveService;
 use Illuminate\Http\Request;
 
 class CardController
 {
-    /**
-     * @var MetadataResolver
-     */
-    private $resolver;
-    /**
-     * @var Formatter
-     */
-    private $formatter;
-
-    public function __construct(MetadataResolver $resolver, Formatter $formatter)
-    {
-        $this->resolver = $resolver;
-        $this->formatter = $formatter;
-    }
-
-    public function show(Request $request)
+    public function show(Request $request, MetadataResolveService $service)
     {
         $request->validate([
             'url:required|url'
         ]);
 
-        $url = $this->formatter->normalizeUrl($request->input('url'));
-
-        $metadata = Metadata::find($url);
-        if ($metadata === null || ($metadata->expires_at !== null && $metadata->expires_at < now())) {
-            $resolved = $this->resolver->resolve($url);
-            $metadata = Metadata::updateOrCreate(['url' => $url], [
-                'title' => $resolved->title,
-                'description' => $resolved->description,
-                'image' => $resolved->image,
-                'expires_at' => $resolved->expires_at
-            ]);
-
-            $tagIds = [];
-            foreach ($resolved->tags as $tagName) {
-                $tag = Tag::firstOrCreate(['name' => $tagName]);
-                $tagIds[] = $tag->id;
-            }
-            $metadata->tags()->sync($tagIds);
+        try {
+            $metadata = $service->execute($request->input('url'));
+        } catch (DeniedHostException $e) {
+            abort(403, $e->getMessage());
         }
-
         $metadata->load('tags');
 
         $response = response($metadata);

app/Http/Controllers/Api/WebhookController.php

@@ -0,0 +1,105 @@
+<?php
+
+namespace App\Http\Controllers\Api;
+
+use App\CheckinWebhook;
+use App\Ejaculation;
+use App\Events\LinkDiscovered;
+use App\Http\Controllers\Controller;
+use App\Http\Resources\EjaculationResource;
+use App\Tag;
+use Carbon\Carbon;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Validator;
+use Illuminate\Validation\ValidationException;
+
+class WebhookController extends Controller
+{
+    public function checkin(CheckinWebhook $webhook, Request $request)
+    {
+        if (!$webhook->isAvailable()) {
+            return response()->json([
+                'status' => 404,
+                'error' => [
+                    'message' => 'The webhook is unavailable'
+                ]
+            ], 404);
+        }
+
+        $validator = Validator::make($request->all(), [
+            'checked_in_at' => 'nullable|date|after_or_equal:2000-01-01 00:00:00|before_or_equal:2099-12-31 23:59:59',
+            'note' => 'nullable|string|max:500',
+            'link' => 'nullable|url|max:2000',
+            'tags' => 'nullable|array',
+            'tags.*' => ['string', 'not_regex:/[\s\r\n]/u', 'max:255'],
+            'is_private' => 'nullable|boolean',
+            'is_too_sensitive' => 'nullable|boolean',
+            'discard_elapsed_time' => 'nullable|boolean',
+        ], [
+            'tags.*.not_regex' => 'The :attribute cannot contain spaces, tabs and newlines.'
+        ]);
+
+        try {
+            $inputs = $validator->validate();
+        } catch (ValidationException $e) {
+            return response()->json([
+                'status' => 422,
+                'error' => [
+                    'message' => 'Validation failed',
+                    'violations' => $validator->errors()->all(),
+                ]
+            ], 422);
+        }
+
+        $ejaculatedDate = empty($inputs['checked_in_at']) ? now() : new Carbon($inputs['checked_in_at']);
+        $ejaculatedDate = $ejaculatedDate->setTimezone(date_default_timezone_get())->startOfMinute();
+        if (Ejaculation::where(['user_id' => $webhook->user_id, 'ejaculated_date' => $ejaculatedDate])->count()) {
+            return response()->json([
+                'status' => 422,
+                'error' => [
+                    'message' => 'Checkin already exists in this time',
+                ]
+            ], 422);
+        }
+
+        $ejaculation = DB::transaction(function () use ($inputs, $webhook, $ejaculatedDate) {
+            $ejaculation = Ejaculation::create([
+                'user_id' => $webhook->user_id,
+                'ejaculated_date' => $ejaculatedDate,
+                'note' => $inputs['note'] ?? '',
+                'link' => $inputs['link'] ?? '',
+                'source' => Ejaculation::SOURCE_WEBHOOK,
+                'is_private' => (bool)($inputs['is_private'] ?? false),
+                'is_too_sensitive' => (bool)($inputs['is_too_sensitive'] ?? false),
+                'discard_elapsed_time' => (bool)($inputs['discard_elapsed_time'] ?? false),
+                'checkin_webhook_id' => $webhook->id
+            ]);
+
+            $tagIds = [];
+            if (!empty($inputs['tags'])) {
+                foreach ($inputs['tags'] as $tag) {
+                    $tag = trim($tag);
+                    if ($tag === '') {
+                        continue;
+                    }
+
+                    $tag = Tag::firstOrCreate(['name' => $tag]);
+                    $tagIds[] = $tag->id;
+                }
+            }
+            $ejaculation->tags()->sync($tagIds);
+
+            return $ejaculation;
+        });
+
+        if (!empty($ejaculation->link)) {
+            event(new LinkDiscovered($ejaculation->link));
+        }
+
+        return response()->json([
+            'status' => 200,
+            'checkin' => new EjaculationResource($ejaculation)
+        ]);
+    }
+}

app/Http/Controllers/Auth/RegisterController.php

@@ -5,6 +5,7 @@ namespace App\Http\Controllers\Auth;
 use App\Http\Controllers\Controller;
 use App\User;
 use Illuminate\Foundation\Auth\RegistersUsers;
+use Illuminate\Support\Facades\Hash;
 use Illuminate\Support\Facades\Validator;
 
 class RegisterController extends Controller
@@ -48,9 +49,9 @@ class RegisterController extends Controller
     protected function validator(array $data)
     {
         $rules = [
-            'name' => 'required|string|regex:/^[a-zA-Z0-9_-]+$/u|max:15|unique:users',
+            'name' => 'required|string|regex:/^[a-zA-Z0-9_-]+$/u|max:15|unique:users|unique:deactivated_users',
             'email' => 'required|string|email|max:255|unique:users',
-            'password' => 'required|string|min:6|confirmed'
+            'password' => 'required|string|min:8|confirmed'
         ];
 
         // reCAPTCHAのキーが設定されている場合、判定を有効化
@@ -78,7 +79,7 @@ class RegisterController extends Controller
             'name' => $data['name'],
             'display_name' => $data['name'],
             'email' => $data['email'],
-            'password' => bcrypt($data['password']),
+            'password' => Hash::make($data['password']),
             'is_protected' => $data['is_protected'] ?? false,
             'accept_analytics' => $data['accept_analytics'] ?? false
         ]);

app/Http/Controllers/Auth/VerificationController.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Http\Controllers\Auth;
+
+use App\Http\Controllers\Controller;
+use Illuminate\Foundation\Auth\VerifiesEmails;
+
+class VerificationController extends Controller
+{
+    /*
+    |--------------------------------------------------------------------------
+    | Email Verification Controller
+    |--------------------------------------------------------------------------
+    |
+    | This controller is responsible for handling email verification for any
+    | user that recently registered with the application. Emails may also
+    | be re-sent if the user didn't receive the original email message.
+    |
+    */
+
+    use VerifiesEmails;
+
+    /**
+     * Where to redirect users after verification.
+     *
+     * @var string
+     */
+    protected $redirectTo = '/home';
+
+    /**
+     * Create a new controller instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        $this->middleware('auth');
+        $this->middleware('signed')->only('verify');
+        $this->middleware('throttle:6,1')->only('verify', 'resend');
+    }
+}

app/Http/Controllers/EjaculationController.php

@@ -9,22 +9,36 @@ use App\User;
 use Carbon\Carbon;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\DB;
 use Validator;
 
 class EjaculationController extends Controller
 {
     public function create(Request $request)
     {
-        $defaults = [
-            'date' => $request->input('date', date('Y/m/d')),
-            'time' => $request->input('time', date('H:i')),
-            'link' => $request->input('link', ''),
-            'tags' => $request->input('tags', ''),
-            'note' => $request->input('note', ''),
-            'is_private' => $request->input('is_private', 0) == 1
+        $tags = old('tags') ?? $request->input('tags', '');
+        if (!empty($tags)) {
+            $tags = explode(' ', $tags);
+        }
+
+        $errors = $request->session()->get('errors');
+        $initialState = [
+            'mode' => 'create',
+            'fields' => [
+                'date' => old('date') ?? $request->input('date', date('Y/m/d')),
+                'time' => old('time') ?? $request->input('time', date('H:i')),
+                'link' => old('link') ?? $request->input('link', ''),
+                'tags' => $tags,
+                'note' => old('note') ?? $request->input('note', ''),
+                'is_private' => old('is_private') ?? $request->input('is_private', 0) == 1,
+                'is_too_sensitive' => old('is_too_sensitive') ?? $request->input('is_too_sensitive', 0) == 1,
+                'is_realtime' => old('is_realtime', true),
+                'discard_elapsed_time' => old('discard_elapsed_time') ?? $request->input('discard_elapsed_time') == 1,
+            ],
+            'errors' => isset($errors) ? $errors->getMessages() : null
         ];
 
-        return view('ejaculation.checkin')->with('defaults', $defaults);
+        return view('ejaculation.checkin')->with('initialState', $initialState);
     }
 
     public function store(Request $request)
@@ -32,15 +46,20 @@ class EjaculationController extends Controller
         $inputs = $request->all();
 
         $validator = Validator::make($inputs, [
-            'date' => 'required|date_format:Y/m/d',
-            'time' => 'required|date_format:H:i',
+            'date' => 'required_without:is_realtime|date_format:Y/m/d',
+            'time' => 'required_without:is_realtime|date_format:H:i',
             'note' => 'nullable|string|max:500',
             'link' => 'nullable|url|max:2000',
             'tags' => 'nullable|string',
         ])->after(function ($validator) use ($request, $inputs) {
             // 日時の重複チェック
             if (!$validator->errors()->hasAny(['date', 'time'])) {
-                $dt = $inputs['date'] . ' ' . $inputs['time'];
+                if (isset($inputs['date']) && isset($inputs['time'])) {
+                    $dt = Carbon::createFromFormat('Y/m/d H:i', $inputs['date'] . ' ' . $inputs['time']);
+                } else {
+                    $dt = now();
+                }
+                $dt = $dt->startOfMinute();
                 if (Ejaculation::where(['user_id' => Auth::id(), 'ejaculated_date' => $dt])->count()) {
                     $validator->errors()->add('datetime', '既にこの日時にチェックインしているため、登録できません。');
                 }
@@ -48,26 +67,45 @@ class EjaculationController extends Controller
         });
 
         if ($validator->fails()) {
-            return redirect()->route('checkin')->withErrors($validator)->withInput();
+            return redirect()->route('checkin')
+                ->withErrors($validator)
+                ->withInput(array_merge(['is_realtime' => false], $request->input()));
         }
 
-        $ejaculation = Ejaculation::create([
-            'user_id' => Auth::id(),
-            'ejaculated_date' => Carbon::createFromFormat('Y/m/d H:i', $inputs['date'] . ' ' . $inputs['time']),
-            'note' => $inputs['note'] ?? '',
-            'link' => $inputs['link'] ?? '',
-            'is_private' => $request->has('is_private') ?? false
-        ]);
-
-        $tagIds = [];
-        if (!empty($inputs['tags'])) {
-            $tags = explode(' ', $inputs['tags']);
-            foreach ($tags as $tag) {
-                $tag = Tag::firstOrCreate(['name' => $tag]);
-                $tagIds[] = $tag->id;
+        $ejaculation = DB::transaction(function () use ($request, $inputs) {
+            if (isset($inputs['date']) && isset($inputs['time'])) {
+                $ejaculatedDate = Carbon::createFromFormat('Y/m/d H:i', $inputs['date'] . ' ' . $inputs['time']);
+            } else {
+                $ejaculatedDate = now();
             }
-        }
-        $ejaculation->tags()->sync($tagIds);
+            $ejaculatedDate = $ejaculatedDate->startOfMinute();
+            $ejaculation = Ejaculation::create([
+                'user_id' => Auth::id(),
+                'ejaculated_date' => $ejaculatedDate,
+                'note' => $inputs['note'] ?? '',
+                'link' => $inputs['link'] ?? '',
+                'source' => Ejaculation::SOURCE_WEB,
+                'is_private' => $request->has('is_private') ?? false,
+                'is_too_sensitive' => $request->has('is_too_sensitive') ?? false,
+                'discard_elapsed_time' => $request->has('discard_elapsed_time') ?? false,
+            ]);
+
+            $tagIds = [];
+            if (!empty($inputs['tags'])) {
+                $tags = explode(' ', $inputs['tags']);
+                foreach ($tags as $tag) {
+                    if ($tag === '') {
+                        continue;
+                    }
+
+                    $tag = Tag::firstOrCreate(['name' => $tag]);
+                    $tagIds[] = $tag->id;
+                }
+            }
+            $ejaculation->tags()->sync($tagIds);
+
+            return $ejaculation;
+        });
 
         if (!empty($ejaculation->link)) {
             event(new LinkDiscovered($ejaculation->link));
@@ -83,34 +121,49 @@ class EjaculationController extends Controller
             ->firstOrFail();
         $user = User::findOrFail($ejaculation->user_id);
 
-        // 1つ前のチェックインからの経過時間を求める
-        $previousEjaculation = Ejaculation::select('ejaculated_date')
-            ->where('user_id', $ejaculation->user_id)
-            ->where('ejaculated_date', '<', $ejaculation->ejaculated_date)
-            ->orderByDesc('ejaculated_date')
-            ->first();
-        if (!empty($previousEjaculation)) {
-            $ejaculatedSpan = $ejaculation->ejaculated_date
-                ->diff($previousEjaculation->ejaculated_date)
-                ->format('%a日 %h時間 %i分');
-        } else {
-            $ejaculatedSpan = null;
-        }
-
-        return view('ejaculation.show')->with(compact('user', 'ejaculation', 'ejaculatedSpan'));
+        return view('ejaculation.show')->with(compact('user', 'ejaculation'));
     }
 
-    public function edit($id)
+    public function edit(Request $request, $id)
     {
         $ejaculation = Ejaculation::findOrFail($id);
 
-        return view('ejaculation.edit')->with(compact('ejaculation'));
+        $this->authorize('edit', $ejaculation);
+
+        if (old('tags') === null) {
+            $tags = $ejaculation->tags->pluck('name');
+        } else {
+            $tags = old('tags');
+            if (!empty($tags)) {
+                $tags = explode(' ', $tags);
+            }
+        }
+
+        $errors = $request->session()->get('errors');
+        $initialState = [
+            'mode' => 'update',
+            'fields' => [
+                'date' => old('date') ?? $ejaculation->ejaculated_date->format('Y/m/d'),
+                'time' => old('time') ?? $ejaculation->ejaculated_date->format('H:i'),
+                'link' => old('link') ?? $ejaculation->link,
+                'tags' => $tags,
+                'note' => old('note') ?? $ejaculation->note,
+                'is_private' => is_bool(old('is_private')) ? old('is_private') : $ejaculation->is_private,
+                'is_too_sensitive' => is_bool(old('is_too_sensitive')) ? old('is_too_sensitive') : $ejaculation->is_too_sensitive,
+                'discard_elapsed_time' => is_bool(old('discard_elapsed_time')) ? old('discard_elapsed_time') : $ejaculation->discard_elapsed_time,
+            ],
+            'errors' => isset($errors) ? $errors->getMessages() : null
+        ];
+
+        return view('ejaculation.edit')->with(compact('ejaculation', 'initialState'));
     }
 
     public function update(Request $request, $id)
     {
         $ejaculation = Ejaculation::findOrFail($id);
 
+        $this->authorize('edit', $ejaculation);
+
         $inputs = $request->all();
 
         $validator = Validator::make($inputs, [
@@ -133,22 +186,30 @@ class EjaculationController extends Controller
             return redirect()->route('checkin.edit', ['id' => $id])->withErrors($validator)->withInput();
         }
 
-        $ejaculation->fill([
-            'ejaculated_date' => Carbon::createFromFormat('Y/m/d H:i', $inputs['date'] . ' ' . $inputs['time']),
-            'note' => $inputs['note'] ?? '',
-            'link' => $inputs['link'] ?? '',
-            'is_private' => $request->has('is_private') ?? false
-        ])->save();
+        DB::transaction(function () use ($ejaculation, $request, $inputs) {
+            $ejaculation->fill([
+                'ejaculated_date' => Carbon::createFromFormat('Y/m/d H:i', $inputs['date'] . ' ' . $inputs['time']),
+                'note' => $inputs['note'] ?? '',
+                'link' => $inputs['link'] ?? '',
+                'is_private' => $request->has('is_private') ?? false,
+                'is_too_sensitive' => $request->has('is_too_sensitive') ?? false,
+                'discard_elapsed_time' => $request->has('discard_elapsed_time') ?? false,
+            ])->save();
 
-        $tagIds = [];
-        if (!empty($inputs['tags'])) {
-            $tags = explode(' ', $inputs['tags']);
-            foreach ($tags as $tag) {
-                $tag = Tag::firstOrCreate(['name' => $tag]);
-                $tagIds[] = $tag->id;
+            $tagIds = [];
+            if (!empty($inputs['tags'])) {
+                $tags = explode(' ', $inputs['tags']);
+                foreach ($tags as $tag) {
+                    if ($tag === '') {
+                        continue;
+                    }
+
+                    $tag = Tag::firstOrCreate(['name' => $tag]);
+                    $tagIds[] = $tag->id;
+                }
             }
-        }
-        $ejaculation->tags()->sync($tagIds);
+            $ejaculation->tags()->sync($tagIds);
+        });
 
         if (!empty($ejaculation->link)) {
             event(new LinkDiscovered($ejaculation->link));
@@ -160,10 +221,21 @@ class EjaculationController extends Controller
     public function destroy($id)
     {
         $ejaculation = Ejaculation::findOrFail($id);
+
+        $this->authorize('edit', $ejaculation);
+
         $user = User::findOrFail($ejaculation->user_id);
-        $ejaculation->tags()->detach();
-        $ejaculation->delete();
+
+        DB::transaction(function () use ($ejaculation) {
+            $ejaculation->tags()->detach();
+            $ejaculation->delete();
+        });
 
         return redirect()->route('user.profile', ['name' => $user->name])->with('status', '削除しました。');
     }
+
+    public function tools()
+    {
+        return view('ejaculation.tools');
+    }
 }

app/Http/Controllers/HomeController.php

@@ -66,11 +66,13 @@ SQL
                 ->where('users.is_protected', false)
                 ->where('ejaculations.is_private', false)
                 ->where('ejaculations.link', '<>', '')
+                ->where('ejaculations.ejaculated_date', '<=', Carbon::now())
                 ->orderBy('ejaculations.ejaculated_date', 'desc')
                 ->select('ejaculations.*')
                 ->with('user', 'tags')
                 ->withLikes()
-                ->take(10)
+                ->visibleToTimeline()
+                ->take(21)
                 ->get();
 
             return view('home')->with(compact('informations', 'categories', 'globalEjaculationCounts', 'publicLinkedEjaculations'));

app/Http/Controllers/SearchController.php

@@ -4,20 +4,30 @@ namespace App\Http\Controllers;
 
 use App\Ejaculation;
 use App\Tag;
+use App\Utilities\Formatter;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
 
 class SearchController extends Controller
 {
+    /** @var Formatter */
+    private $formatter;
+
+    public function __construct(Formatter $formatter)
+    {
+        $this->formatter = $formatter;
+    }
+
     public function index(Request $request)
     {
         $inputs = $request->validate([
             'q' => 'required'
         ]);
 
+        $q = $this->normalizeQuery($inputs['q']);
         $results = Ejaculation::query()
-            ->whereHas('tags', function ($query) use ($inputs) {
-                $query->where('name', 'like', "%{$inputs['q']}%");
+            ->whereHas('tags', function ($query) use ($q) {
+                $query->where('normalized_name', 'like', "%{$q}%");
             })
             ->whereHas('user', function ($query) {
                 $query->where('is_protected', false);
@@ -41,11 +51,17 @@ class SearchController extends Controller
             'q' => 'required'
         ]);
 
+        $q = $this->normalizeQuery($inputs['q']);
         $results = Tag::query()
-            ->where('name', 'like', "%{$inputs['q']}%")
+            ->where('normalized_name', 'like', "%{$q}%")
             ->paginate(50)
             ->appends($inputs);
 
         return view('search.relatedTag')->with(compact('inputs', 'results'));
     }
+
+    private function normalizeQuery(string $query): string
+    {
+        return $this->formatter->normalizeTagName($query);
+    }
 }

app/Http/Controllers/SettingController.php

@@ -2,9 +2,19 @@
 
 namespace App\Http\Controllers;
 
+use App\CheckinWebhook;
+use App\DeactivatedUser;
+use App\Ejaculation;
+use App\Exceptions\CsvImportException;
+use App\Services\CheckinCsvExporter;
+use App\Services\CheckinCsvImporter;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Hash;
 use Illuminate\Support\Facades\Validator;
+use Illuminate\Validation\Rule;
+use Illuminate\Validation\ValidationException;
 
 class SettingController extends Controller
 {
@@ -18,10 +28,18 @@ class SettingController extends Controller
         $inputs = $request->all();
         $validator = Validator::make($inputs, [
             'display_name' => 'required|string|max:20',
+            'email' => [
+                'required',
+                'string',
+                'email',
+                'max:255',
+                Rule::unique('users')->ignore(Auth::user()->email, 'email')
+            ],
             'bio' => 'nullable|string|max:160',
             'url' => 'nullable|url|max:2000'
         ], [], [
             'display_name' => '名前',
+            'email' => 'メールアドレス',
             'bio' => '自己紹介',
             'url' => 'URL'
         ]);
@@ -32,6 +50,7 @@ class SettingController extends Controller
 
         $user = Auth::user();
         $user->display_name = $inputs['display_name'];
+        $user->email = $inputs['email'];
         $user->bio = $inputs['bio'] ?? '';
         $user->url = $inputs['url'] ?? '';
         $user->save();
@@ -57,6 +76,164 @@ class SettingController extends Controller
         return redirect()->route('setting.privacy')->with('status', 'プライバシー設定を更新しました。');
     }
 
+    public function webhooks()
+    {
+        $webhooks = Auth::user()->checkinWebhooks;
+        $webhooksLimit = CheckinWebhook::PER_USER_LIMIT;
+
+        return view('setting.webhooks')->with(compact('webhooks', 'webhooksLimit'));
+    }
+
+    public function storeWebhooks(Request $request)
+    {
+        $validated = $request->validate([
+            'name' => [
+                'required',
+                'string',
+                'max:255',
+                Rule::unique('checkin_webhooks', 'name')->where(function ($query) {
+                    return $query->where('user_id', Auth::id());
+                })
+            ]
+        ], [], [
+            'name' => '名前'
+        ]);
+
+        if (Auth::user()->checkinWebhooks()->count() >= CheckinWebhook::PER_USER_LIMIT) {
+            return redirect()->route('setting.webhooks')
+                ->with('status', CheckinWebhook::PER_USER_LIMIT . '件以上のWebhookを作成することはできません。');
+        }
+
+        Auth::user()->checkinWebhooks()->create($validated);
+
+        return redirect()->route('setting.webhooks')->with('status', '作成しました。');
+    }
+
+    public function destroyWebhooks(CheckinWebhook $webhook)
+    {
+        $webhook->delete();
+
+        return redirect()->route('setting.webhooks')->with('status', '削除しました。');
+    }
+
+    public function import()
+    {
+        return view('setting.import');
+    }
+
+    public function storeImport(Request $request)
+    {
+        $validated = $request->validate([
+            'file' => 'required|file'
+        ], [], [
+            'file' => 'ファイル'
+        ]);
+
+        $file = $request->file('file');
+        if (!$file->isValid()) {
+            return redirect()->route('setting.import')->withErrors(['file' => 'ファイルのアップロードに失敗しました。']);
+        }
+
+        try {
+            set_time_limit(0);
+
+            $importer = new CheckinCsvImporter(Auth::user(), $file->path());
+            $imported = $importer->execute();
+
+            return redirect()->route('setting.import')->with('status', "{$imported}件のインポートに性交しました。");
+        } catch (CsvImportException $e) {
+            return redirect()->route('setting.import')->with('import_errors', $e->getErrors());
+        }
+    }
+
+    public function destroyImport()
+    {
+        Auth::user()
+            ->ejaculations()
+            ->where('ejaculations.source', Ejaculation::SOURCE_CSV)
+            ->delete();
+
+        return redirect()->route('setting.import')->with('status', '削除が完了しました。');
+    }
+
+    public function export()
+    {
+        return view('setting.export');
+    }
+
+    public function exportToCsv(Request $request)
+    {
+        $validated = $request->validate([
+            'charset' => ['required', Rule::in(['utf8', 'sjis'])]
+        ]);
+
+        $charsets = [
+            'utf8' => 'UTF-8',
+            'sjis' => 'SJIS-win'
+        ];
+
+        $filename = tempnam(sys_get_temp_dir(), 'tissue_export_tmp_');
+        try {
+            // 気休め
+            set_time_limit(0);
+
+            $exporter = new CheckinCsvExporter(Auth::user(), $filename, $charsets[$validated['charset']]);
+            $exporter->execute();
+        } catch (\Throwable $e) {
+            unlink($filename);
+            throw $e;
+        }
+
+        return response()
+            ->download($filename, 'TissueCheckin_' . date('Y-m-d_H-i-s') . '.csv')
+            ->deleteFileAfterSend(true);
+    }
+
+    public function deactivate()
+    {
+        return view('setting.deactivate');
+    }
+
+    public function destroyUser(Request $request)
+    {
+        // パスワードチェック
+        $validated = $request->validate([
+            'password' => 'required|string'
+        ]);
+
+        if (!Hash::check($validated['password'], Auth::user()->getAuthPassword())) {
+            throw ValidationException::withMessages([
+                'password' => 'パスワードが正しくありません。'
+            ]);
+        }
+
+        // データの削除
+        set_time_limit(0);
+        DB::transaction(function () {
+            $user = Auth::user();
+
+            // 関連レコードの削除
+            // TODO: 別にDELETE文相当のクエリを一発発行するだけでもいい？
+            foreach ($user->ejaculations as $ejaculation) {
+                $ejaculation->delete();
+            }
+            foreach ($user->likes as $like) {
+                $like->delete();
+            }
+
+            // 先にログアウトしないとユーザーは消せない
+            Auth::logout();
+
+            // ユーザーの削除
+            $user->delete();
+
+            // ユーザー名履歴に追記
+            DeactivatedUser::create(['name' => $user->name]);
+        });
+
+        return view('setting.deactivated');
+    }
+
     // ( ◠‿◠ )☛ここに気づいたか・・・消えてもらう ▂▅▇█▓▒░(’ω’)░▒▓█▇▅▂うわあああああああ
 //    public function password()
 //    {

app/Http/Controllers/TagController.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Tag;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\DB;
+
+class TagController extends Controller
+{
+    public function index()
+    {
+        $tags = Tag::select(DB::raw(
+            <<<'SQL'
+tags.name,
+count(*) AS "checkins_count"
+SQL
+        ))
+            ->join('ejaculation_tag', 'tags.id', '=', 'ejaculation_tag.tag_id')
+            ->join('ejaculations', 'ejaculations.id', '=', 'ejaculation_tag.ejaculation_id')
+            ->join('users', 'users.id', '=', 'ejaculations.user_id')
+            ->where('ejaculations.is_private', false)
+            ->where(function ($query) {
+                $query->where('users.is_protected', false);
+                if (Auth::check()) {
+                    $query->orWhere('users.id', Auth::id());
+                }
+            })
+            ->groupBy('tags.name')
+            ->orderByDesc('checkins_count')
+            ->orderBy('tags.name')
+            ->paginate(100);
+
+        return view('tag.index', compact('tags'));
+    }
+}

app/Http/Controllers/TimelineController.php

@@ -4,6 +4,7 @@ namespace App\Http\Controllers;
 
 use App\Ejaculation;
 use Illuminate\Http\Request;
+use Illuminate\Support\Carbon;
 
 class TimelineController extends Controller
 {
@@ -13,10 +14,12 @@ class TimelineController extends Controller
             ->where('users.is_protected', false)
             ->where('ejaculations.is_private', false)
             ->where('ejaculations.link', '<>', '')
+            ->where('ejaculations.ejaculated_date', '<=', Carbon::now())
             ->orderBy('ejaculations.ejaculated_date', 'desc')
             ->select('ejaculations.*')
             ->with('user', 'tags')
             ->withLikes()
+            ->visibleToTimeline()
             ->paginate(21);
 
         return view('timeline.public')->with(compact('ejaculations'));

app/Http/Controllers/UserController.php

@@ -5,9 +5,11 @@ namespace App\Http\Controllers;
 use App\Ejaculation;
 use App\User;
 use Carbon\Carbon;
+use Carbon\CarbonInterface;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\DB;
+use Validator;
 
 class UserController extends Controller
 {
@@ -26,15 +28,19 @@ class UserController extends Controller
         // チェックインの取得
         $query = Ejaculation::select(DB::raw(
             <<<'SQL'
-id,
+ejaculations.id,
 ejaculated_date,
 note,
 is_private,
+is_too_sensitive,
 link,
-to_char(lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC), 'YYYY/MM/DD HH24:MI') AS before_date,
-to_char(ejaculated_date - (lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)), 'FMDDD日 FMHH24時間 FMMI分') AS ejaculated_span
+source,
+discard_elapsed_time,
+to_char(before_dates.before_date, 'YYYY/MM/DD HH24:MI') AS before_date,
+to_char(ejaculated_date - before_dates.before_date, 'FMDDD日 FMHH24時間 FMMI分') AS ejaculated_span
 SQL
         ))
+            ->joinSub($this->queryBeforeEjaculatedDates(), 'before_dates', 'before_dates.id', '=', 'ejaculations.id')
             ->where('user_id', $user->id);
         if (!Auth::check() || $user->id !== Auth::id()) {
             $query = $query->where('is_private', false);
@@ -58,7 +64,18 @@ SQL
             ->limit(10)
             ->get();
 
-        return view('user.profile')->with(compact('user', 'ejaculations', 'tags'));
+        // シコ草
+        $countByDayQuery = $this->countEjaculationByDay($user)
+            ->where('ejaculated_date', '>=', now()->startOfMonth()->subMonths(9))
+            ->where('ejaculated_date', '<', now()->addMonth()->startOfMonth())
+            ->get();
+        $countByDay = [];
+        foreach ($countByDayQuery as $data) {
+            $date = Carbon::createFromFormat('Y/m/d', $data->date);
+            $countByDay[$date->timestamp] = $data->count;
+        }
+
+        return view('user.profile')->with(compact('user', 'ejaculations', 'tags', 'countByDay'));
     }
 
     public function stats($name)
@@ -68,18 +85,161 @@ SQL
             abort(404);
         }
 
-        $dateUntil = now()->addMonth()->startOfMonth();
+        $availableMonths = $this->makeStatsAvailableMonths($user);
+        $graphData = $this->makeGraphData($user);
 
-        $groupByDay = Ejaculation::select(DB::raw(
+        return view('user.stats.index')->with(compact('user', 'graphData', 'availableMonths'));
+    }
+
+    public function statsYearly($name, $year)
+    {
+        $user = User::where('name', $name)->first();
+        if (empty($user)) {
+            abort(404);
+        }
+
+        $validator = Validator::make(compact('year'), [
+            'year' => 'required|date_format:Y'
+        ]);
+        if ($validator->fails()) {
+            return redirect()->route('user.stats', compact('name'));
+        }
+
+        $availableMonths = $this->makeStatsAvailableMonths($user);
+        if (!isset($availableMonths[$year])) {
+            return redirect()->route('user.stats', compact('name'));
+        }
+
+        $graphData = $this->makeGraphData(
+            $user,
+            Carbon::createFromDate($year, 1, 1, config('app.timezone'))->startOfDay(),
+            Carbon::createFromDate($year, 1, 1, config('app.timezone'))->addYear()->startOfDay()
+        );
+
+        return view('user.stats.yearly')
+            ->with(compact('user', 'graphData', 'availableMonths'))
+            ->with('currentYear', (int) $year);
+    }
+
+    public function statsMonthly($name, $year, $month)
+    {
+        $user = User::where('name', $name)->first();
+        if (empty($user)) {
+            abort(404);
+        }
+
+        $validator = Validator::make(compact('year', 'month'), [
+            'year' => 'required|date_format:Y',
+            'month' => 'required|date_format:m'
+        ]);
+        if ($validator->fails()) {
+            return redirect()->route('user.stats.yearly', compact('name', 'year'));
+        }
+
+        $availableMonths = $this->makeStatsAvailableMonths($user);
+        if (!isset($availableMonths[$year]) || !in_array($month, $availableMonths[$year], false)) {
+            return redirect()->route('user.stats.yearly', compact('name', 'year'));
+        }
+
+        $graphData = $this->makeGraphData(
+            $user,
+            Carbon::createFromDate($year, $month, 1, config('app.timezone'))->startOfDay(),
+            Carbon::createFromDate($year, $month, 1, config('app.timezone'))->addMonth()->startOfDay()
+        );
+
+        return view('user.stats.monthly')
+            ->with(compact('user', 'graphData', 'availableMonths'))
+            ->with('currentYear', (int) $year)
+            ->with('currentMonth', (int) $month);
+    }
+
+    public function okazu($name)
+    {
+        $user = User::where('name', $name)->first();
+        if (empty($user)) {
+            abort(404);
+        }
+
+        // チェックインの取得
+        $query = Ejaculation::select(DB::raw(
             <<<'SQL'
-to_char(ejaculated_date, 'YYYY/MM/DD') AS "date",
-count(*) AS "count"
+ejaculations.id,
+ejaculated_date,
+note,
+is_private,
+is_too_sensitive,
+link,
+source,
+discard_elapsed_time,
+to_char(before_dates.before_date, 'YYYY/MM/DD HH24:MI') AS before_date,
+to_char(ejaculated_date - before_dates.before_date, 'FMDDD日 FMHH24時間 FMMI分') AS ejaculated_span
 SQL
         ))
+            ->joinSub($this->queryBeforeEjaculatedDates(), 'before_dates', 'before_dates.id', '=', 'ejaculations.id')
             ->where('user_id', $user->id)
-            ->where('ejaculated_date', '<', $dateUntil)
-            ->groupBy(DB::raw("to_char(ejaculated_date, 'YYYY/MM/DD')"))
-            ->orderBy(DB::raw("to_char(ejaculated_date, 'YYYY/MM/DD')"))
+            ->where('link', '<>', '');
+        if (!Auth::check() || $user->id !== Auth::id()) {
+            $query = $query->where('is_private', false);
+        }
+        $ejaculations = $query->orderBy('ejaculated_date', 'desc')
+            ->with('tags')
+            ->withLikes()
+            ->paginate(20);
+
+        return view('user.profile')->with(compact('user', 'ejaculations'));
+    }
+
+    public function likes($name)
+    {
+        $user = User::where('name', $name)->first();
+        if (empty($user)) {
+            abort(404);
+        }
+
+        $likes = $user->likes()
+            ->orderBy('created_at', 'desc')
+            ->with('ejaculation.user', 'ejaculation.tags')
+            ->whereHas('ejaculation', function ($query) {
+                $query->where('user_id', Auth::id())
+                    ->orWhere('is_private', false);
+            })
+            ->paginate(20);
+
+        return view('user.likes')->with(compact('user', 'likes'));
+    }
+
+    private function makeStatsAvailableMonths(User $user): array
+    {
+        $availableMonths = [];
+        $oldest = $user->ejaculations()->orderBy('ejaculated_date')->first();
+        if (isset($oldest)) {
+            $oldestMonth = $oldest->ejaculated_date->startOfMonth();
+            $currentMonth = now()->startOfMonth();
+            for ($month = $currentMonth; $oldestMonth <= $currentMonth; $month = $month->subMonth()) {
+                if (!isset($availableMonths[$month->year])) {
+                    $availableMonths[$month->year] = [];
+                }
+                $availableMonths[$month->year][] = $month->month;
+            }
+        }
+
+        return $availableMonths;
+    }
+
+    private function makeGraphData(User $user, CarbonInterface $dateSince = null, CarbonInterface $dateUntil = null): array
+    {
+        if ($dateUntil === null) {
+            $dateUntil = now()->addMonth()->startOfMonth();
+        }
+        $dateCondition = [
+            ['ejaculated_date', '<', $dateUntil],
+        ];
+        if ($dateSince !== null) {
+            $dateCondition[] = ['ejaculated_date', '>=', $dateSince];
+        }
+
+        $groupByDay = $this->countEjaculationByDay($user)
+            ->where($dateCondition)
             ->get();
 
         $groupByHour = Ejaculation::select(DB::raw(
@@ -89,7 +249,7 @@ count(*) AS "count"
 SQL
         ))
             ->where('user_id', $user->id)
-            ->where('ejaculated_date', '<', $dateUntil)
+            ->where($dateCondition)
             ->groupBy(DB::raw("to_char(ejaculated_date, 'HH24')"))
             ->orderBy(DB::raw('1'))
             ->get();
@@ -124,7 +284,7 @@ SQL
             $hourlySum[$hour] += $data->count;
         }
 
-        $graphData = [
+        return [
             'dailySum' => $dailySum,
             'dowSum' => $dowSum,
             'monthlySum' => $monthlySum,
@@ -133,57 +293,28 @@ SQL
             'hourlyKey' => array_keys($hourlySum),
             'hourlySum' => array_values($hourlySum),
         ];
-
-        return view('user.stats')->with(compact('user', 'graphData'));
     }
 
-    public function okazu($name)
+    private function countEjaculationByDay(User $user)
     {
-        $user = User::where('name', $name)->first();
-        if (empty($user)) {
-            abort(404);
-        }
-
-        // チェックインの取得
-        $query = Ejaculation::select(DB::raw(
+        return Ejaculation::select(DB::raw(
             <<<'SQL'
-id,
-ejaculated_date,
-note,
-is_private,
-link,
-to_char(lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC), 'YYYY/MM/DD HH24:MI') AS before_date,
-to_char(ejaculated_date - (lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)), 'FMDDD日 FMHH24時間 FMMI分') AS ejaculated_span
+to_char(ejaculated_date, 'YYYY/MM/DD') AS "date",
+count(*) AS "count"
 SQL
         ))
             ->where('user_id', $user->id)
-            ->where('link', '<>', '');
-        if (!Auth::check() || $user->id !== Auth::id()) {
-            $query = $query->where('is_private', false);
-        }
-        $ejaculations = $query->orderBy('ejaculated_date', 'desc')
-            ->with('tags')
-            ->paginate(20);
-
-        return view('user.profile')->with(compact('user', 'ejaculations'));
+            ->groupBy(DB::raw("to_char(ejaculated_date, 'YYYY/MM/DD')"))
+            ->orderBy(DB::raw("to_char(ejaculated_date, 'YYYY/MM/DD')"));
     }
 
-    public function likes($name)
+    private function queryBeforeEjaculatedDates()
     {
-        $user = User::where('name', $name)->first();
-        if (empty($user)) {
-            abort(404);
-        }
-
-        $likes = $user->likes()
-            ->orderBy('created_at', 'desc')
-            ->with('ejaculation.user', 'ejaculation.tags')
-            ->whereHas('ejaculation', function ($query) {
-                $query->where('user_id', Auth::id())
-                    ->orWhere('is_private', false);
-            })
-            ->paginate(20);
-
-        return view('user.likes')->with(compact('user', 'likes'));
+        return DB::table('ejaculations')->selectRaw(
+            <<<'SQL'
+id,
+(select ejaculated_date from ejaculations e2 where e2.ejaculated_date < ejaculations.ejaculated_date and e2.user_id = ejaculations.user_id order by e2.ejaculated_date desc limit 1) AS before_date
+SQL
+        );
     }
 }

app/Http/Kernel.php

@@ -14,11 +14,11 @@ class Kernel extends HttpKernel
      * @var array
      */
     protected $middleware = [
-        \Illuminate\Foundation\Http\Middleware\CheckForMaintenanceMode::class,
+        \App\Http\Middleware\TrustProxies::class,
+        \App\Http\Middleware\CheckForMaintenanceMode::class,
         \Illuminate\Foundation\Http\Middleware\ValidatePostSize::class,
         \App\Http\Middleware\TrimStrings::class,
         \Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull::class,
-        \App\Http\Middleware\TrustProxies::class,
     ];
 
     /**
@@ -38,15 +38,17 @@ class Kernel extends HttpKernel
             \App\Http\Middleware\NormalizeLineEnding::class,
         ],
 
-        // 現時点では内部APIしかないので、認証の手間を省くためにステートフルにしている。
         'api' => [
+            \App\Http\Middleware\EnforceJson::class,
+            \Illuminate\Routing\Middleware\SubstituteBindings::class,
+        ],
+
+        'stateful' => [
             \App\Http\Middleware\EncryptCookies::class,
             \Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse::class,
             \Illuminate\Session\Middleware\StartSession::class,
             \App\Http\Middleware\VerifyCsrfToken::class,
-            'throttle:60,1',
-            'bindings',
-        ],
+        ]
     ];
 
     /**
@@ -57,11 +59,33 @@ class Kernel extends HttpKernel
      * @var array
      */
     protected $routeMiddleware = [
-        'auth' => \Illuminate\Auth\Middleware\Authenticate::class,
+        'auth' => \App\Http\Middleware\Authenticate::class,
         'auth.basic' => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
         'bindings' => \Illuminate\Routing\Middleware\SubstituteBindings::class,
+        'cache.headers' => \Illuminate\Http\Middleware\SetCacheHeaders::class,
         'can' => \Illuminate\Auth\Middleware\Authorize::class,
         'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
+        'password.confirm' => \Illuminate\Auth\Middleware\RequirePassword::class,
+        'signed' => \Illuminate\Routing\Middleware\ValidateSignature::class,
         'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
+        'verified' => \Illuminate\Auth\Middleware\EnsureEmailIsVerified::class,
+    ];
+
+    /**
+     * The priority-sorted list of middleware.
+     *
+     * This forces non-global middleware to always be in the given order.
+     *
+     * @var array
+     */
+    protected $middlewarePriority = [
+        \App\Http\Middleware\EncryptCookies::class,
+        \Illuminate\Session\Middleware\StartSession::class,
+        \Illuminate\View\Middleware\ShareErrorsFromSession::class,
+        \App\Http\Middleware\Authenticate::class,
+        \Illuminate\Routing\Middleware\ThrottleRequests::class,
+        \Illuminate\Session\Middleware\AuthenticateSession::class,
+        \Illuminate\Routing\Middleware\SubstituteBindings::class,
+        \Illuminate\Auth\Middleware\Authorize::class,
     ];
 }

app/Http/Middleware/Authenticate.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use Illuminate\Auth\Middleware\Authenticate as Middleware;
+
+class Authenticate extends Middleware
+{
+    /**
+     * Get the path the user should be redirected to when they are not authenticated.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @return string|null
+     */
+    protected function redirectTo($request)
+    {
+        if (! $request->expectsJson()) {
+            return route('login');
+        }
+    }
+}

app/Http/Middleware/CheckForMaintenanceMode.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use Illuminate\Foundation\Http\Middleware\CheckForMaintenanceMode as Middleware;
+
+class CheckForMaintenanceMode extends Middleware
+{
+    /**
+     * The URIs that should be reachable while maintenance mode is enabled.
+     *
+     * @var array
+     */
+    protected $except = [
+        //
+    ];
+}

app/Http/Middleware/EnforceJson.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use Closure;
+
+/**
+ * Request headerに Accept: application/json を上書きする。APIエンドポイント用。
+ */
+class EnforceJson
+{
+    /**
+     * Handle an incoming request.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @param  \Closure  $next
+     * @return mixed
+     */
+    public function handle($request, Closure $next)
+    {
+        $request->headers->set('Accept', 'application/json');
+
+        return $next($request);
+    }
+}

app/Http/Middleware/TrustProxies.php

@@ -10,20 +10,14 @@ class TrustProxies extends Middleware
     /**
      * The trusted proxies for this application.
      *
-     * @var array
+     * @var array|string
      */
     protected $proxies = '**';
 
     /**
-     * The current proxy header mappings.
+     * The headers that should be used to detect proxies.
      *
-     * @var array
+     * @var int
      */
-    protected $headers = [
-        Request::HEADER_FORWARDED => 'FORWARDED',
-        Request::HEADER_X_FORWARDED_FOR => 'X_FORWARDED_FOR',
-        Request::HEADER_X_FORWARDED_HOST => 'X_FORWARDED_HOST',
-        Request::HEADER_X_FORWARDED_PORT => 'X_FORWARDED_PORT',
-        Request::HEADER_X_FORWARDED_PROTO => 'X_FORWARDED_PROTO',
-    ];
+    protected $headers = Request::HEADER_X_FORWARDED_ALL;
 }

app/Http/Middleware/VerifyCsrfToken.php

@@ -6,6 +6,13 @@ use Illuminate\Foundation\Http\Middleware\VerifyCsrfToken as BaseVerifier;
 
 class VerifyCsrfToken extends BaseVerifier
 {
+    /**
+     * Indicates whether the XSRF-TOKEN cookie should be set on the response.
+     *
+     * @var bool
+     */
+    protected $addHttpCookie = true;
+
     /**
      * The URIs that should be excluded from CSRF verification.
      *

app/Http/Resources/EjaculationResource.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Http\Resources;
+
+use Illuminate\Http\Resources\Json\JsonResource;
+
+class EjaculationResource extends JsonResource
+{
+    /**
+     * Transform the resource into an array.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @return array
+     */
+    public function toArray($request)
+    {
+        return [
+            'id' => $this->id,
+            'checked_in_at' => $this->ejaculated_date->format(\DateTime::ATOM),
+            'note' => $this->note,
+            'link' => $this->link,
+            'tags' => $this->tags->pluck('name'),
+            'source' => $this->source,
+            'is_private' => $this->is_private,
+            'is_too_sensitive' => $this->is_too_sensitive,
+        ];
+    }
+}

app/Http/ViewComposers/ProfileStatsComposer.php

@@ -19,6 +19,7 @@ class ProfileStatsComposer
         if (!$view->offsetExists('user')) {
             throw new \LogicException('View data "user" was not exist.');
         }
+        /** @var \App\User $user */
         $user = $view->offsetGet('user');
 
         // 現在のオナ禁セッションの経過時間
@@ -35,35 +36,44 @@ class ProfileStatsComposer
         }
 
         // 概況欄のデータ取得
-        $average = DB::select(<<<'SQL'
+        $average = 0;
+        $divisor = 0;
+        $averageSources = DB::select(<<<'SQL'
 SELECT
-  avg(span) AS average
+  extract(epoch from ejaculated_date - lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)) AS span,
+  discard_elapsed_time
 FROM
-  (
-    SELECT
-      extract(epoch from ejaculated_date - lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)) AS span
-    FROM
-      ejaculations
-    WHERE
-      user_id = :user_id
-    ORDER BY
-      ejaculated_date DESC
-    LIMIT
-      30
-  ) AS temp
+  ejaculations
+WHERE
+  user_id = :user_id
+ORDER BY
+  ejaculated_date DESC
+LIMIT
+  30
 SQL
             , ['user_id' => $user->id]);
+        foreach ($averageSources as $item) {
+            // 経過時間記録対象外のレコードがあったら、それより古いデータは平均の計算に加えない
+            if ($item->discard_elapsed_time) {
+                break;
+            }
+            $average += $item->span;
+            $divisor++;
+        }
+        if ($divisor > 0) {
+            $average /= $divisor;
+        }
 
         $summary = DB::select(<<<'SQL'
 SELECT
   max(span) AS longest,
   min(span) AS shortest,
-  sum(span) AS total_times,
-  count(*) AS total_checkins
+  sum(span) AS total_times
 FROM
   (
     SELECT
-      extract(epoch from ejaculated_date - lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)) AS span
+      extract(epoch from ejaculated_date - lead(ejaculated_date, 1, NULL) OVER (ORDER BY ejaculated_date DESC)) AS span,
+      discard_elapsed_time
     FROM
       ejaculations
     WHERE
@@ -71,9 +81,13 @@ FROM
     ORDER BY
       ejaculated_date DESC
   ) AS temp
+WHERE
+  discard_elapsed_time = FALSE
 SQL
             , ['user_id' => $user->id]);
 
-        $view->with(compact('latestEjaculation', 'currentSession', 'average', 'summary'));
+        $total = $user->ejaculations()->count();
+
+        $view->with(compact('latestEjaculation', 'currentSession', 'average', 'summary', 'total'));
     }
 }

app/Listeners/LinkCollector.php

@@ -3,32 +3,24 @@
 namespace App\Listeners;
 
 use App\Events\LinkDiscovered;
-use App\Metadata;
-use App\MetadataResolver\MetadataResolver;
-use App\Tag;
-use App\Utilities\Formatter;
-use GuzzleHttp\Exception\TransferException;
+use App\MetadataResolver\DeniedHostException;
+use App\Services\MetadataResolveService;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Support\Facades\Log;
 
 class LinkCollector
 {
-    /** @var Formatter */
-    private $formatter;
-    /** @var MetadataResolver */
-    private $metadataResolver;
+    /** @var MetadataResolveService */
+    private $metadataResolveService;
 
     /**
      * Create the event listener.
      *
-     * @param Formatter $formatter
-     * @param MetadataResolver $metadataResolver
+     * @param MetadataResolveService $metadataResolveService
      */
-    public function __construct(Formatter $formatter, MetadataResolver $metadataResolver)
+    public function __construct(MetadataResolveService $metadataResolveService)
     {
-        $this->formatter = $formatter;
-        $this->metadataResolver = $metadataResolver;
+        $this->metadataResolveService = $metadataResolveService;
     }
 
     /**
@@ -39,33 +31,13 @@ class LinkCollector
      */
     public function handle(LinkDiscovered $event)
     {
-        // URLの正規化
-        $url = $this->formatter->normalizeUrl($event->url);
-
-        // 無かったら取得
-        // TODO: ある程度古かったら再取得とかありだと思う
-        $metadata = Metadata::find($url);
-        if ($metadata == null || ($metadata->expires_at !== null && $metadata->expires_at < now())) {
-            try {
-                $resolved = $this->metadataResolver->resolve($url);
-                $metadata = Metadata::updateOrCreate(['url' => $url], [
-                    'title' => $resolved->title,
-                    'description' => $resolved->description,
-                    'image' => $resolved->image,
-                    'expires_at' => $resolved->expires_at
-                ]);
-
-                $tagIds = [];
-                foreach ($resolved->tags as $tagName) {
-                    $tag = Tag::firstOrCreate(['name' => $tagName]);
-                    $tagIds[] = $tag->id;
-                }
-                $metadata->tags()->sync($tagIds);
-            } catch (TransferException $e) {
-                // 何らかの通信エラーによってメタデータの取得に失敗した時、とりあえずエラーログにURLを残す
-                Log::error(self::class . ': メタデータの取得に失敗 URL=' . $url);
-                report($e);
-            }
+        try {
+            $this->metadataResolveService->execute($event->url);
+        } catch (DeniedHostException $e) {
+            // ignored
+        } catch (\Exception $e) {
+            // 今のところこのイベントは同期実行されるので、上流をクラッシュさせないために雑catchする
+            report($e);
         }
     }
 }

app/Metadata.php

@@ -2,6 +2,8 @@
 
 namespace App;
 
+use Carbon\CarbonInterface;
+use GuzzleHttp\Exception\RequestException;
 use Illuminate\Database\Eloquent\Model;
 
 class Metadata extends Model
@@ -13,10 +15,66 @@ class Metadata extends Model
     protected $fillable = ['url', 'title', 'description', 'image', 'expires_at'];
     protected $visible = ['url', 'title', 'description', 'image', 'expires_at', 'tags'];
 
-    protected $dates = ['created_at', 'updated_at', 'expires_at'];
+    protected $dates = ['created_at', 'updated_at', 'expires_at', 'error_at'];
 
     public function tags()
     {
         return $this->belongsToMany(Tag::class)->withTimestamps();
     }
+
+    public function needRefresh(): bool
+    {
+        return $this->isExpired() || $this->error_at !== null;
+    }
+
+    public function isExpired(): bool
+    {
+        return $this->expires_at !== null && $this->expires_at < now();
+    }
+
+    public function storeException(CarbonInterface $error_at, \Exception $exception): self
+    {
+        $this->prepareFieldsOnError();
+        $this->error_at = $error_at;
+        $this->error_exception_class = get_class($exception);
+        $this->error_body = $exception->getMessage();
+        if ($exception instanceof RequestException) {
+            $this->error_http_code = $exception->getCode();
+        } else {
+            $this->error_http_code = null;
+        }
+        $this->error_count++;
+
+        return $this;
+    }
+
+    public function storeError(CarbonInterface $error_at, string $body, ?int $httpCode = null): self
+    {
+        $this->prepareFieldsOnError();
+        $this->error_at = $error_at;
+        $this->error_exception_class = null;
+        $this->error_body = $body;
+        $this->error_http_code = $httpCode;
+        $this->error_count++;
+
+        return $this;
+    }
+
+    public function clearError(): self
+    {
+        $this->error_at = null;
+        $this->error_exception_class = null;
+        $this->error_body = null;
+        $this->error_http_code = null;
+        $this->error_count = 0;
+
+        return $this;
+    }
+
+    private function prepareFieldsOnError()
+    {
+        $this->title = $this->title ?? '';
+        $this->description = $this->description ?? '';
+        $this->image = $this->image ?? '';
+    }
 }

app/MetadataResolver/ActivityPubResolver.php

@@ -37,6 +37,10 @@ class ActivityPubResolver implements Resolver, Parser
         $activityOrObject = json_decode($json, true);
         $object = $activityOrObject['object'] ?? $activityOrObject;
 
+        if ($object['type'] !== 'Note') {
+            throw new UnsupportedContentException('Unsupported object type: ' . $object['type']);
+        }
+
         $metadata = new Metadata();
 
         $metadata->title = isset($object['attributedTo']) ? $this->getTitleFromActor($object['attributedTo']) : '';

app/MetadataResolver/CienResolver.php

@@ -4,6 +4,7 @@ namespace App\MetadataResolver;
 
 use Carbon\Carbon;
 use GuzzleHttp\Client;
+use Symfony\Component\DomCrawler\Crawler;
 
 class CienResolver extends MetadataResolver
 {
@@ -25,19 +26,28 @@ class CienResolver extends MetadataResolver
     public function resolve(string $url): Metadata
     {
         $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $html = (string) $res->getBody();
+        $metadata = $this->ogpResolver->parse($html);
+        $crawler = new Crawler($html);
 
-            // 画像URLから有効期限の起点を拾う
-            parse_str(parse_url($metadata->image, PHP_URL_QUERY), $params);
-            if (empty($params['px-time'])) {
-                throw new \RuntimeException('Parameter "px-time" not found. Image=' . $metadata->image . ' Source=' . $url);
-            }
-            $metadata->expires_at = Carbon::createFromTimestamp($params['px-time'])->addHour(1);
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        // OGPのデフォルトはバナーなので、投稿に使える画像があればそれを使う
+        $selector = 'img[data-actual*="image-web"]';
+        if ($crawler->filter($selector)->count() !== 0) {
+            $metadata->image = $crawler->filter($selector)->attr('data-actual');
         }
+
+        // JWTがついていれば画像URLのJWTから有効期限を拾う
+        parse_str(parse_url($metadata->image, PHP_URL_QUERY), $params);
+        if (isset($params['jwt'])) {
+            $parts = explode('.', $params['jwt']);
+            if (count($parts) !== 3) {
+                throw new \RuntimeException('Invalid jwt. Image=' . $metadata->image . ' Source=' . $url);
+            }
+            $payload = json_decode(base64_decode(str_replace(['-', '_'], ['+', '/'], $parts[1])), true);
+
+            $metadata->expires_at = Carbon::createFromTimestamp($payload['exp']);
+        }
+
+        return $metadata;
     }
 }

app/MetadataResolver/DLsiteResolver.php

@@ -46,12 +46,29 @@ class DLsiteResolver implements Resolver
 
         // 重複削除
         $tags = array_values(array_unique($tags));
+        sort($tags);
 
         return $tags;
     }
 
     public function resolve(string $url): Metadata
     {
+        //アフィリエイトの場合は普通のURLに変換
+        // ID型
+        if (preg_match('~/dlaf/=(/.+/.+)?/link/~', $url)) {
+            preg_match('~www\.dlsite\.com/(?P<genre>.+)/dlaf/=(/.+/.+)?/link/work/aid/(?P<AffiliateId>.+)/id/(?P<titleId>..\d+)(\.html)?~', $url, $matches);
+            $url = "https://www.dlsite.com/{$matches['genre']}/work/=/product_id/{$matches['titleId']}.html";
+        }
+        // URL型
+        if (strpos($url, '/dlaf/=/aid/') !== false) {
+            preg_match('~www\.dlsite\.com/.+/dlaf/=/aid/.+/url/(?P<url>.+)~', $url, $matches);
+            $affiliateUrl = urldecode($matches['url']);
+            if (preg_match('~www\.dlsite\.com/.+/(work|announce)/=/product_id/..\d+(\.html)?~', $affiliateUrl, $matches)) {
+                $url = $affiliateUrl;
+            } else {
+                throw new \RuntimeException("アフィリエイト先のリンクがDLsiteのタイトルではありません: $affiliateUrl");
+            }
+        }
 
         //スマホページの場合はPCページに正規化
         if (strpos($url, '-touch') !== false) {
@@ -59,57 +76,55 @@ class DLsiteResolver implements Resolver
         }
 
         $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $metadata = $this->ogpResolver->parse($res->getBody());
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
+        $dom = new \DOMDocument();
+        @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
+        $xpath = new \DOMXPath($dom);
 
-            // OGPタイトルから[]に囲まれているmakerを取得する
-            // 複数の作者がいる場合スペース区切りになるためexplodeしている
-            // スペースを含むmakerの場合名前の一部しか取れないが動作には問題ない
-            preg_match('~ \[([^\[\]]*)\] (予告作品 )?\| DLsite(がるまに)?$~', $metadata->title, $match);
-            $makers = explode(' ', $match[1]);
+        // OGPタイトルから[]に囲まれているmakerを取得する
+        // 複数の作者がいる場合スペース区切りになるためexplodeしている
+        // スペースを含むmakerの場合名前の一部しか取れないが動作には問題ない
+        preg_match('~ \[([^\[\]]*)\] (予告作品 )?\| DLsite(がるまに)?$~', $metadata->title, $match);
+        $makers = explode(' ', $match[1]);
 
-            //フォローボタン(.btn_follow)はテキストを含んでしまうことがあるので要素を削除しておく
-            $followButtonNode = $xpath->query('//*[@class="btn_follow"]')->item(0);
-            $followButtonNode->parentNode->removeChild($followButtonNode);
+        //フォローボタン(.add_follow)はテキストを含んでしまうことがあるので要素を削除しておく
+        $followButtonNode = $xpath->query('//*[@class="add_follow"]')->item(0);
+        $followButtonNode->parentNode->removeChild($followButtonNode);
 
-            // maker, makerHeadを探す
+        // maker, makerHeadを探す
 
-            // makers
-            // #work_makerから「makerを含むテキスト」を持つ要素を持つtdを探す
-            // 作者名単体の場合もあるし、"作者A / 作者B"のようになることもある
-            $makersNode = $xpath->query('//*[@id="work_maker"]//*[contains(text(), "' . $makers[0] . '")]/ancestor::td')->item(0);
-            $makers = trim($makersNode->textContent);
+        // makers
+        // #work_makerから「makerを含むテキスト」を持つ要素を持つtdを探す
+        // 作者名単体の場合もあるし、"作者A / 作者B"のようになることもある
+        $makersNode = $xpath->query('//*[@id="work_maker"]//*[contains(text(), "' . $makers[0] . '")]/ancestor::td')->item(0);
+        // nbspをspaceに置換
+        $makers = trim(str_replace("\xc2\xa0", ' ', $makersNode->textContent));
 
-            // makersHaed
-            // $makerNode(td)に対するthを探す
-            // "著者", "サークル名", "ブランド名"など
-            $makersHeadNode = $xpath->query('preceding-sibling::th', $makersNode)->item(0);
-            $makersHead = trim($makersHeadNode->textContent);
+        // makersHaed
+        // $makerNode(td)に対するthを探す
+        // "著者", "サークル名", "ブランド名"など
+        $makersHeadNode = $xpath->query('preceding-sibling::th', $makersNode)->item(0);
+        $makersHead = trim($makersHeadNode->textContent);
 
-            // 余分な文を消す
+        // 余分な文を消す
 
-            // OGPタイトルから作者名とサイト名を消す
-            $metadata->title = trim(preg_replace('~ \[[^\[\]]*\] (予告作品 )?\| DLsite(がるまに)?$~', '', $metadata->title));
+        // OGPタイトルから作者名とサイト名を消す
+        $metadata->title = trim(preg_replace('~ \[[^\[\]]*\] (予告作品 )?\| DLsite(がるまに)?$~', '', $metadata->title));
 
-            // OGP説明文から定型文を消す
-            if (strpos($url, 'dlsite.com/eng/') || strpos($url, 'dlsite.com/ecchi-eng/')) {
-                $metadata->description = trim(preg_replace('~DLsite.+ is a download shop for .+With a huge selection of products, we\'re sure you\'ll find whatever tickles your fancy\. DLsite is one of the greatest indie contents download shops in Japan\.$~', '', $metadata->description));
-            } else {
-                $metadata->description = trim(preg_replace('~「DLsite.+」は.+のダウンロードショップ。お気に入りの作品をすぐダウンロードできてすぐ楽しめる！毎日更新しているのであなたが探している作品にきっと出会えます。国内最大級の二次元総合ダウンロードショップ「DLsite」！$~', '', $metadata->description));
-            }
-
-            // 整形
-            $metadata->description = $makersHead . ': ' . $makers . PHP_EOL . $metadata->description;
-            $metadata->image = str_replace('img_sam.jpg', 'img_main.jpg', $metadata->image);
-            $metadata->tags = $this->extractTags($res->getBody());
-
-            return $metadata;
+        // OGP説明文から定型文を消す
+        if (strpos($url, 'dlsite.com/eng/') || strpos($url, 'dlsite.com/ecchi-eng/')) {
+            $metadata->description = preg_replace('~DLsite.+ is a download shop for .+With a huge selection of products, we\'re sure you\'ll find whatever tickles your fancy\. DLsite is one of the greatest indie contents download shops in Japan\.$~', '', $metadata->description);
         } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+            $metadata->description = preg_replace('~「DLsite.+」は.+のダウンロードショップ。お気に入りの作品をすぐダウンロードできてすぐ楽しめる！毎日更新しているのであなたが探している作品にきっと出会えます。国内最大級の二次元総合ダウンロードショップ「DLsite」！$~', '', $metadata->description);
         }
+        $metadata->description = trim(strip_tags($metadata->description));
+
+        // 整形
+        $metadata->description = $makersHead . ': ' . $makers . PHP_EOL . $metadata->description;
+        $metadata->image = str_replace('img_sam.jpg', 'img_main.jpg', $metadata->image);
+        $metadata->tags = $this->extractTags($res->getBody());
+
+        return $metadata;
     }
 }

app/MetadataResolver/DeniedHostException.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use Exception;
+use Throwable;
+
+/**
+ * メタデータの解決を禁止しているホストに対して取得を試み、ブロックされたことを表します。
+ */
+class DeniedHostException extends Exception
+{
+    private $url;
+
+    public function __construct(string $url, Throwable $previous = null)
+    {
+        parent::__construct("Access denied by system policy: $url", 0, $previous);
+        $this->url = $url;
+    }
+
+    public function getUrl(): string
+    {
+        return $this->url;
+    }
+
+    public function getHost(): string
+    {
+        return parse_url($this->url, PHP_URL_HOST);
+    }
+}

app/MetadataResolver/DeviantArtResolver.php

@@ -23,35 +23,17 @@ class DeviantArtResolver implements Resolver
 
     public function resolve(string $url): Metadata
     {
-        $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $res = $this->client->get('https://backend.deviantart.com/oembed?url=' . $url);
+        $data = json_decode($res->getBody()->getContents(), true);
+        $metadata = new Metadata();
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
-
-            $node = $xpath->query('//*[@id="pimp-preload"]/following-sibling::div//img')->item(0);
-            $srcset = $node->getAttribute('srcset');
-            $srcset_array = explode('w,', $srcset);
-            $src = end($srcset_array);
-            $src = preg_replace('~ \d+w$~', '', $src);
-
-            if (preg_match('~\.wixmp\.com$~', parse_url($src)['host'])) {
-                // アスペクト比を保ったまま、縦か横が最大700pxになるように変換する。
-                // Ref: https://support.wixmp.com/en/article/image-service-3835799
-                if (strpos($src, '/v1/fill/')) {
-                    $src = preg_replace('~/v1/fill/w_\d+,h_\d+,q_\d+,strp~', '/v1/fit/w_700,h_700,q_70,strp', $src);
-                } else {
-                    $src = $src . '/v1/fit/w_700,h_700,q_70,strp/image.jpg';
-                }
-            }
-
-            $metadata->image = $src;
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        $metadata->title = $data['title'] ?? '';
+        $metadata->description = 'By ' . $data['author_name'];
+        $metadata->image  = $data['url'];
+        if (isset($data['tags'])) {
+            $metadata->tags = explode(', ', $data['tags']);
         }
+
+        return $metadata;
     }
 }

app/MetadataResolver/DisallowedByProviderException.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use RuntimeException;
+use Throwable;
+
+/**
+ * ContentProviderの提供するrobots.txtによってクロールが拒否された場合にスローされます。
+ */
+class DisallowedByProviderException extends RuntimeException
+{
+    private $url;
+
+    public function __construct(string $url, Throwable $previous = null)
+    {
+        parent::__construct("Access denied by robots.txt: $url", 0, $previous);
+        $this->url = $url;
+    }
+
+    public function getUrl(): string
+    {
+        return $this->url;
+    }
+
+    public function getHost(): string
+    {
+        return parse_url($this->url, PHP_URL_HOST);
+    }
+}

app/MetadataResolver/FC2ContentsResolver.php

@@ -1,44 +0,0 @@
-<?php
-
-namespace App\MetadataResolver;
-
-use GuzzleHttp\Client;
-
-class FC2ContentsResolver implements Resolver
-{
-    /**
-     * @var Client
-     */
-    private $client;
-    /**
-     * @var OGPResolver
-     */
-    private $ogpResolver;
-
-    public function __construct(Client $client, OGPResolver $ogpResolver)
-    {
-        $this->client = $client;
-        $this->ogpResolver = $ogpResolver;
-    }
-
-    public function resolve(string $url): Metadata
-    {
-        $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
-
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
-
-            $thumbnailNode = $xpath->query('//*[@class="main_thum_img"]/a')->item(0);
-            if ($thumbnailNode) {
-                $metadata->image = preg_replace('~^http:~', 'https:', $thumbnailNode->getAttribute('href'));
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
-        }
-    }
-}

app/MetadataResolver/FantiaResolver.php

@@ -3,7 +3,6 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
-use Illuminate\Support\Facades\Log;
 
 class FantiaResolver implements Resolver
 {
@@ -11,46 +10,31 @@ class FantiaResolver implements Resolver
      * @var Client
      */
     private $client;
-    /**
-     * @var OGPResolver
-     */
-    private $ogpResolver;
 
-    public function __construct(Client $client, OGPResolver $ogpResolver)
+    public function __construct(Client $client)
     {
         $this->client = $client;
-        $this->ogpResolver = $ogpResolver;
     }
 
     public function resolve(string $url): Metadata
     {
-        preg_match("~\d+~", $url, $match);
-        $postId = $match[0];
+        preg_match("~posts/(\d+)~", $url, $match);
+        $postId = $match[1];
 
-        $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $res = $this->client->get("https://fantia.jp/api/v1/posts/{$postId}");
+        $data = json_decode(str_replace('\r\n', '\n', (string) $res->getBody()), true);
+        $post = $data['post'];
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
+        $tags = array_map(function ($tag) {
+            return $tag['name'];
+        }, $post['tags']);
 
-            $node = $xpath->query("//meta[@property='twitter:image']")->item(0);
-            $ogpUrl = $node->getAttribute('content');
+        $metadata = new Metadata();
+        $metadata->title = $post['title'] ?? '';
+        $metadata->description = 'サークル: ' . $post['fanclub']['fanclub_name_with_creator_name'] . PHP_EOL . $post['comment'];
+        $metadata->image = str_replace('micro', 'main', $post['thumb_micro']) ?? '';
+        $metadata->tags = array_merge($tags, [$post['fanclub']['creator_name']]);
 
-            // 投稿に画像がない場合（ogp.jpgでない場合）のみ大きい画像に変換する
-            if ($ogpUrl != 'http://fantia.jp/images/ogp.jpg') {
-                preg_match("~https://fantia\.s3\.amazonaws\.com/uploads/post/file/{$postId}/ogp_(.*?)\.(jpg|png)~", $ogpUrl, $match);
-                $uuid = $match[1];
-                $extension = $match[2];
-
-                // 大きい画像に変換
-                $metadata->image = "https://c.fantia.jp/uploads/post/file/{$postId}/main_{$uuid}.{$extension}";
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
-        }
+        return $metadata;
     }
 }

app/MetadataResolver/FanzaResolver.php

@@ -3,6 +3,8 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
+use GuzzleHttp\Cookie\CookieJar;
+use Symfony\Component\DomCrawler\Crawler;
 
 class FanzaResolver implements Resolver
 {
@@ -21,17 +23,86 @@ class FanzaResolver implements Resolver
         $this->ogpResolver = $ogpResolver;
     }
 
+    /**
+     * arrayの各要素をtrim・スペースの_置換をした後、重複した値を削除してキーを詰め直す
+     *
+     * @param array $array
+     *
+     * @return array 処理されたarray
+     */
+    public function array_finish(array $array): array
+    {
+        $array = array_map('trim', $array);
+        $array = array_map((function ($value) {
+            return str_replace(' ', '_', $value);
+        }), $array);
+        $array = array_unique($array);
+        $array = array_values($array);
+
+        return $array;
+    }
+
     public function resolve(string $url): Metadata
     {
-        $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
-            $metadata->image = preg_replace("~(pr|ps)\.jpg$~", 'pl.jpg', $metadata->image);
-            $metadata->description = str_replace('<>', '', $metadata->description);
+        $cookieJar = CookieJar::fromArray(['age_check_done' => '1'], 'dmm.co.jp');
+
+        $res = $this->client->get($url, ['cookies' => $cookieJar]);
+        $html = (string) $res->getBody();
+        $crawler = new Crawler($html);
+
+        // 動画
+        if (preg_match('~www\.dmm\.co\.jp/digital/(videoa|videoc|anime)/-/detail~', $url)) {
+            $metadata = new Metadata();
+            $metadata->title = trim($crawler->filter('#title')->text(''));
+            $metadata->description = trim(strip_tags(str_replace('【FANZA(ファンザ)】', '', $crawler->filter('meta[name="description"]')->attr('content'))));
+            $metadata->image = preg_replace("~(pr|ps)\.jpg$~", 'pl.jpg', $crawler->filter('meta[property="og:image"]')->attr('content'));
+            $metadata->tags = $this->array_finish($crawler->filter('.box-rank+table a[href*="list/=/article="]')->extract(['_text']));
 
             return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
         }
+
+        // 同人
+        if (mb_strpos($url, 'www.dmm.co.jp/dc/doujin/-/detail/') !== false) {
+            $genre = $this->array_finish($crawler->filter('.m-productInformation a:not([href="#update-top"])')->extract(['_text']));
+            $genre = array_filter($genre, (function ($text) {
+                return !preg_match('~％OFF対象$~', $text);
+            }));
+
+            $metadata = new Metadata();
+            $metadata->title = $crawler->filter('meta[property="og:title"]')->attr('content');
+            $metadata->description = trim($crawler->filter('.summary__txt')->text(''));
+            $metadata->image = $crawler->filter('meta[property="og:image"]')->attr('content');
+            $metadata->tags = array_merge($genre, [$crawler->filter('.circleName__txt')->text('')]);
+
+            return $metadata;
+        }
+
+        // 電子書籍
+        if (mb_strpos($url, 'book.dmm.co.jp/detail/') !== false) {
+            $metadata = new Metadata();
+            $metadata->title = trim($crawler->filter('#title')->text(''));
+            $metadata->description = trim($crawler->filter('.m-boxDetailProduct__info__story')->text(''));
+            $metadata->image = preg_replace("~(pr|ps)\.jpg$~", 'pl.jpg', $crawler->filter('meta[property="og:image"]')->attr('content'));
+            $metadata->tags = $this->array_finish($crawler->filter('.m-boxDetailProductInfoMainList__description__list__item, .m-boxDetailProductInfo__list__description__item a')->extract(['_text']));
+
+            return $metadata;
+        }
+
+        // PCゲーム
+        if (mb_strpos($url, 'dlsoft.dmm.co.jp/detail/') !== false) {
+            $metadata = new Metadata();
+            $metadata->title = trim($crawler->filter('#title')->text(''));
+            $metadata->description = trim($crawler->filter('.area-detail-read .text-overflow')->text(''));
+            $metadata->image = preg_replace("~(pr|ps)\.jpg$~", 'pl.jpg', $crawler->filter('meta[property="og:image"]')->attr('content'));
+            $metadata->tags = $this->array_finish($crawler->filter('.container02 table a[href*="list/article="]')->extract(['_text']));
+
+            return $metadata;
+        }
+
+        // 上で特に対応しなかったURL 画像の置換くらいはしておく
+        $metadata = $this->ogpResolver->parse($html);
+        $metadata->image = preg_replace("~(pr|ps)\.jpg$~", 'pl.jpg', $metadata->image);
+
+        return $metadata;
     }
 }

app/MetadataResolver/HentaiFoundryResolver.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Cookie\CookieJar;
+use Symfony\Component\DomCrawler\Crawler;
+
+class HentaiFoundryResolver implements Resolver
+{
+    /**
+     * @var Client
+     */
+    private $client;
+    /**
+     * @var OGPResolver
+     */
+    private $ogpResolver;
+
+    public function __construct(Client $client, OGPResolver $ogpResolver)
+    {
+        $this->client = $client;
+        $this->ogpResolver = $ogpResolver;
+    }
+
+    private function nbsp2space(string $string): string
+    {
+        return str_replace("\xc2\xa0", ' ', $string);
+    }
+
+    private function br2nl(string $string): string
+    {
+        return str_replace('<br>', PHP_EOL, $string);
+    }
+
+    public function resolve(string $url): Metadata
+    {
+        $res = $this->client->get(
+            http_build_url($url, ['query' => 'enterAgree=1']),
+            ['cookies' => new CookieJar()]
+        );
+
+        $metadata = new Metadata();
+        $crawler = new Crawler((string) $res->getBody());
+
+        $author =  $crawler->filter('#picBox .boxtitle a')->text();
+        $description = trim(strip_tags($this->nbsp2space($this->br2nl($crawler->filter('.picDescript')->html()))));
+
+        $metadata->title = $crawler->filter('#picBox .boxtitle .imageTitle')->text();
+        $metadata->description = 'by ' . $author . PHP_EOL . $description;
+        $metadata->image = 'https:' . $crawler->filter('img[src^="//picture"]')->attr('src');
+        $metadata->tags = $crawler->filter('a[rel="tag"]')->extract('_text');
+
+        return $metadata;
+    }
+}

app/MetadataResolver/IwaraResolver.php

@@ -3,6 +3,7 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
+use Symfony\Component\DomCrawler\Crawler;
 
 class IwaraResolver implements Resolver
 {
@@ -19,51 +20,41 @@ class IwaraResolver implements Resolver
     public function resolve(string $url): Metadata
     {
         $res = $this->client->get($url);
+        $metadata = new Metadata();
+        $html = (string) $res->getBody();
+        $crawler = new Crawler($html);
 
-        if ($res->getStatusCode() === 200) {
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
+        $infoElements = $crawler->filter('#video-player + div, .field-name-field-video-url + div, .field-name-field-images + div');
+        $title = $infoElements->filter('h1.title')->text();
+        $author = $infoElements->filter('.username')->text();
+        $description = $infoElements->filter('.field-type-text-with-summary')->text('');
+        $tags =  $infoElements->filter('a[href^="/videos"], a[href^="/images"]')->extract('_text');
+        // 役に立たないタグを削除する
+        $tags = array_values(array_diff($tags, ['Uncategorized', 'Other']));
+        array_push($tags, $author);
 
-            $metadata = new Metadata();
+        $metadata->title = $title;
+        $metadata->description = '投稿者: ' . $author . PHP_EOL . trim($description);
+        $metadata->tags = $tags;
 
-            // find title
-            foreach ($xpath->query('//title') as $node) {
-                $content = $node->textContent;
-                if (!empty($content)) {
-                    $metadata->title = $content;
-                    break;
-                }
-            }
-
-            // find thumbnail
-            foreach ($xpath->query('//*[@id="video-player"]') as $node) {
-                $poster = $node->getAttribute('poster');
-                if (!empty($poster)) {
-                    if (strpos($poster, '//') === 0) {
-                        $poster = 'https:' . $poster;
-                    }
-                    $metadata->image = $poster;
-                    break;
-                }
-            }
-            if (empty($metadata->image)) {
-                // YouTube embedded?
-                foreach ($xpath->query('//div[@class="embedded-video"]//iframe') as $node) {
-                    $src = $node->getAttribute('src');
-                    if (preg_match('~youtube\.com/embed/(\S+)\?~', $src, $matches) !== -1) {
-                        $youtubeId = $matches[1];
-                        $iwaraThumbUrl = 'https://i.iwara.tv/sites/default/files/styles/thumbnail/public/video_embed_field_thumbnails/youtube/' . $youtubeId . '.jpg';
-
-                        $metadata->image = $iwaraThumbUrl;
-                        break;
-                    }
-                }
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        // iwara video
+        if ($crawler->filter('#video-player')->count()) {
+            $metadata->image = 'https:' . $crawler->filter('#video-player')->attr('poster');
         }
+
+        // youtube
+        if ($crawler->filter('iframe[src^="//www.youtube.com"]')->count()) {
+            if (preg_match('~youtube\.com/embed/(\S+)\?~', $crawler->filter('iframe[src^="//www.youtube.com"]')->attr('src'), $matches) === 1) {
+                $youtubeId = $matches[1];
+                $metadata->image = 'https://img.youtube.com/vi/' . $youtubeId . '/maxresdefault.jpg';
+            }
+        }
+
+        // images
+        if ($crawler->filter('.field-name-field-images')->count()) {
+            $metadata->image = 'https:' . $crawler->filter('.field-name-field-images a')->first()->attr('href');
+        }
+
+        return $metadata;
     }
 }

app/MetadataResolver/Kb10uyShortStoryServerResolver.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use GuzzleHttp\Client;
+use Symfony\Component\DomCrawler\Crawler;
+
+class Kb10uyShortStoryServerResolver implements Resolver
+{
+    protected const EXCLUDED_TAGS = ['R-15', 'R-18'];
+
+    /**
+     * @var Client
+     */
+    private $client;
+
+    public function __construct(Client $client)
+    {
+        $this->client = $client;
+    }
+
+    public function resolve(string $url): Metadata
+    {
+        $res = $this->client->get($url);
+        $html = (string) $res->getBody();
+        $crawler = new Crawler($html);
+        $infoElement = $crawler->filter('div.post-info');
+
+        $metadata = new Metadata();
+        $metadata->title = $infoElement->filter('h1')->text();
+        $metadata->description = trim($infoElement->filter('p.summary')->text());
+        $metadata->tags = array_values(array_diff($infoElement->filter('ul.tags > li.tag > a')->extract('_text'), self::EXCLUDED_TAGS));
+
+        return $metadata;
+    }
+}

app/MetadataResolver/KomifloResolver.php

@@ -24,33 +24,31 @@ class KomifloResolver implements Resolver
         $id = $matches[1];
 
         $res = $this->client->get('https://api.komiflo.com/content/id/' . $id);
-        if ($res->getStatusCode() === 200) {
-            $json = json_decode($res->getBody()->getContents(), true);
-            $metadata = new Metadata();
+        $json = json_decode($res->getBody()->getContents(), true);
+        $metadata = new Metadata();
 
-            $metadata->title = $json['content']['data']['title'] ?? '';
-            $metadata->description = ($json['content']['attributes']['artists']['children'][0]['data']['name'] ?? '?') .
-                ' - ' .
-                ($json['content']['parents'][0]['data']['title'] ?? '?');
-            $metadata->image = 'https://t.komiflo.com/564_mobile_large_3x/' . $json['content']['named_imgs']['cover']['filename'];
+        $metadata->title = $json['content']['data']['title'] ?? '';
+        $metadata->description = ($json['content']['attributes']['artists']['children'][0]['data']['name'] ?? '?') .
+            ' - ' . ($json['content']['parents'][0]['data']['title'] ?? '?');
+        $metadata->image = 'https://t.komiflo.com/564_mobile_large_3x/' . $json['content']['named_imgs']['cover']['filename'];
 
-            // 作者情報
-            if (!empty($json['content']['attributes']['artists']['children'])) {
-                foreach ($json['content']['attributes']['artists']['children'] as $artist) {
-                    $metadata->tags[] = preg_replace('/\s/', '_', $artist['data']['name']);
-                }
+        // 作者情報
+        if (!empty($json['content']['attributes']['artists']['children'])) {
+            foreach ($json['content']['attributes']['artists']['children'] as $artist) {
+                $metadata->tags[] = preg_replace('/\s/', '_', $artist['data']['name']);
             }
-
-            // タグ
-            if (!empty($json['content']['attributes']['tags']['children'])) {
-                foreach ($json['content']['attributes']['tags']['children'] as $tag) {
-                    $metadata->tags[] = preg_replace('/\s/', '_', $tag['data']['name']);
-                }
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
         }
+
+        // タグ
+        if (!empty($json['content']['attributes']['tags']['children'])) {
+            $tags = [];
+            foreach ($json['content']['attributes']['tags']['children'] as $tag) {
+                $tags[] = preg_replace('/\s/', '_', $tag['data']['name']);
+            }
+            sort($tags);
+            $metadata->tags = array_merge($metadata->tags, $tags);
+        }
+
+        return $metadata;
     }
 }

app/MetadataResolver/MelonbooksResolver.php

@@ -27,46 +27,42 @@ class MelonbooksResolver implements Resolver
         $cookieJar = CookieJar::fromArray(['AUTH_ADULT' => '1'], 'www.melonbooks.co.jp');
 
         $res = $this->client->get($url, ['cookies' => $cookieJar]);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $metadata = $this->ogpResolver->parse($res->getBody());
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
-            $descriptionNodelist = $xpath->query('//div[@id="description"]//p');
-            $specialDescriptionNodelist = $xpath->query('//div[@id="special_description"]//p');
+        $dom = new \DOMDocument();
+        @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
+        $xpath = new \DOMXPath($dom);
+        $descriptionNodelist = $xpath->query('//div[@id="description"]//p');
+        $specialDescriptionNodelist = $xpath->query('//div[@id="special_description"]//p');
 
-            // censoredフラグの除去
-            if (mb_strpos($metadata->image, '&c=1') !== false) {
-                $metadata->image = preg_replace('/&c=1/u', '', $metadata->image);
-            }
-
-            // 抽出
-            preg_match('~^(.+)（(.+)）の通販・購入はメロンブックス$~', $metadata->title, $match);
-            $title = $match[1];
-            $maker = $match[2];
-
-            // 整形
-            $description = 'サークル: ' . $maker . "\n";
-
-            if ($specialDescriptionNodelist->length !== 0) {
-                $description .= trim(str_replace('<br>', "\n", $specialDescriptionNodelist->item(0)->nodeValue)) . "\n";
-                if ($specialDescriptionNodelist->length === 2) {
-                    $description .= "\n";
-                    $description .= trim(str_replace('<br>', "\n", $specialDescriptionNodelist->item(1)->nodeValue)) . "\n";
-                }
-            }
-
-            if ($descriptionNodelist->length !== 0) {
-                $description .= trim(str_replace('<br>', "\n", $descriptionNodelist->item(0)->nodeValue));
-            }
-
-            $metadata->title = $title;
-            $metadata->description = trim($description);
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        // censoredフラグの除去
+        if (mb_strpos($metadata->image, '&c=1') !== false) {
+            $metadata->image = preg_replace('/&c=1/u', '', $metadata->image);
         }
+
+        // 抽出
+        preg_match('~^(.+)（(.+)）の通販・購入はメロンブックス$~', $metadata->title, $match);
+        $title = $match[1];
+        $maker = $match[2];
+
+        // 整形
+        $description = 'サークル: ' . $maker . "\n";
+
+        if ($specialDescriptionNodelist->length !== 0) {
+            $description .= trim(str_replace('<br>', "\n", $specialDescriptionNodelist->item(0)->nodeValue)) . "\n";
+            if ($specialDescriptionNodelist->length === 2) {
+                $description .= "\n";
+                $description .= trim(str_replace('<br>', "\n", $specialDescriptionNodelist->item(1)->nodeValue)) . "\n";
+            }
+        }
+
+        if ($descriptionNodelist->length !== 0) {
+            $description .= trim(str_replace('<br>', "\n", $descriptionNodelist->item(0)->nodeValue));
+        }
+
+        $metadata->title = $title;
+        $metadata->description = trim($description);
+
+        return $metadata;
     }
 }

app/MetadataResolver/Metadata.php

@@ -23,4 +23,30 @@ class Metadata
      * チェックインタグと同様に保存されるため、スペースや改行文字を含めてはいけません。
      */
     public $tags = [];
+
+    /**
+     * 重複を排除し、正規化を行ったタグの集合を返します。
+     * @return string[]
+     */
+    public function normalizedTags(): array
+    {
+        $tags = [];
+        foreach ($this->tags as $tag) {
+            $tag = $this->sanitize($tag);
+            $tag = $this->trim($tag);
+            $tags[$tag] = true;
+        }
+
+        return array_keys($tags);
+    }
+
+    private function sanitize(string $value): string
+    {
+        return preg_replace('/\r?\n/u', ' ', $value);
+    }
+
+    private function trim(string $value): string
+    {
+        return trim($value);
+    }
 }

app/MetadataResolver/MetadataResolver.php

@@ -14,21 +14,26 @@ class MetadataResolver implements Resolver
         '~komiflo\.com(/#!)?/comics/(\\d+)~' => KomifloResolver::class,
         '~www\.melonbooks\.co\.jp/detail/detail\.php~' => MelonbooksResolver::class,
         '~ec\.toranoana\.(jp|shop)/(tora|joshi)(_[rd]+)?/(ec|digi)/item/~' => ToranoanaResolver::class,
-        '~iwara\.tv/videos/.*~' => IwaraResolver::class,
+        '~iwara\.tv/(videos|images)/.*~' => IwaraResolver::class,
         '~www\.dlsite\.com/.*/(work|announce)/=/product_id/..\d+(\.html)?~' => DLsiteResolver::class,
+        '~www\.dlsite\.com/.*/dlaf/=(/.+/.+)?/link/work/aid/.+(/id)?/..\d+(\.html)?~' => DLsiteResolver::class,
+        '~www\.dlsite\.com/.*/dlaf/=/aid/.+/url/.+~' => DLsiteResolver::class,
         '~dlsite\.jp/...tw/..\d+~' => DLsiteResolver::class,
         '~www\.pixiv\.net/member_illust\.php\?illust_id=\d+~' => PixivResolver::class,
+        '~www\.pixiv\.net/(en/)?artworks/\d+~' => PixivResolver::class,
         '~www\.pixiv\.net/user/\d+/series/\d+~' => PixivResolver::class,
         '~fantia\.jp/posts/\d+~' => FantiaResolver::class,
         '~dmm\.co\.jp/~' => FanzaResolver::class,
         '~www\.patreon\.com/~' => PatreonResolver::class,
         '~www\.deviantart\.com/.*/art/.*~' => DeviantArtResolver::class,
-        '~\.syosetu\.com/n\d+[a-z]{2,}~' => NarouResolver::class,
-        '~ci-en\.jp/creator/\d+/article/\d+~' => CienResolver::class,
+        '~\.syosetu\.com/n\d+[a-z]+~' => NarouResolver::class,
+        '~ci-en\.(jp|net|dlsite\.com)/creator/\d+/article/\d+~' => CienResolver::class,
         '~www\.plurk\.com\/p\/.*~' => PlurkResolver::class,
-        '~(adult\.)?contents\.fc2\.com\/article_search\.php\?id=\d+~' => FC2ContentsResolver::class,
         '~store\.steampowered\.com/app/\d+~' => SteamResolver::class,
         '~www\.xtube\.com/video-watch/.*-\d+$~'=> XtubeResolver::class,
+        '~ss\.kb10uy\.org/posts/\d+$~' => Kb10uyShortStoryServerResolver::class,
+        '~www\.hentai-foundry\.com/pictures/user/.+/\d+/.+~'=> HentaiFoundryResolver::class,
+        '~(www\.)?((mobile|m)\.)?twitter\.com/(#!/)?[0-9a-zA-Z_]{1,15}/status(es)?/([0-9]+)/?(\\?.+)?$~' => TwitterResolver::class,
     ];
 
     public $mimeTypes = [
@@ -44,16 +49,19 @@ class MetadataResolver implements Resolver
     {
         foreach ($this->rules as $pattern => $class) {
             if (preg_match($pattern, $url) === 1) {
-                /** @var Resolver $resolver */
-                $resolver = app($class);
+                try {
+                    /** @var Resolver $resolver */
+                    $resolver = app($class);
 
-                return $resolver->resolve($url);
+                    return $resolver->resolve($url);
+                } catch (UnsupportedContentException $e) {
+                }
             }
         }
 
-        $result = $this->resolveWithAcceptHeader($url);
-        if ($result !== null) {
-            return $result;
+        try {
+            return $this->resolveWithAcceptHeader($url);
+        } catch (UnsupportedContentException $e) {
         }
 
         if (isset($this->defaultResolver)) {
@@ -66,7 +74,7 @@ class MetadataResolver implements Resolver
         throw new \UnexpectedValueException('URL not matched.');
     }
 
-    public function resolveWithAcceptHeader(string $url): ?Metadata
+    public function resolveWithAcceptHeader(string $url): Metadata
     {
         try {
             // Rails等はAcceptに */* が入っていると、ブラウザの適当なAcceptヘッダだと判断して全部無視してしまう。
@@ -111,6 +119,6 @@ class MetadataResolver implements Resolver
             // 5xx は変なAcceptが原因かもしれない（？）ので無視してフォールバック
         }
 
-        return null;
+        throw new UnsupportedContentException();
     }
 }

app/MetadataResolver/NarouResolver.php

@@ -27,34 +27,30 @@ class NarouResolver implements Resolver
         $cookieJar = CookieJar::fromArray(['over18' => 'yes'], '.syosetu.com');
 
         $res = $this->client->get($url, ['cookies' => $cookieJar]);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
-            $metadata->description = '';
+        $metadata = $this->ogpResolver->parse($res->getBody());
+        $metadata->description = '';
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'ASCII,JIS,UTF-8,eucJP-win,SJIS-win'));
-            $xpath = new \DOMXPath($dom);
+        $dom = new \DOMDocument();
+        @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'ASCII,JIS,UTF-8,eucJP-win,SJIS-win'));
+        $xpath = new \DOMXPath($dom);
 
-            $description = [];
+        $description = [];
 
-            // 作者名
-            $writerNodes = $xpath->query('//*[contains(@class, "novel_writername")]');
-            if ($writerNodes->length !== 0 && !empty($writerNodes->item(0)->textContent)) {
-                $description[] = trim($writerNodes->item(0)->textContent);
-            }
-
-            // あらすじ
-            $exNodes = $xpath->query('//*[@id="novel_ex"]');
-            if ($exNodes->length !== 0 && !empty($exNodes->item(0)->textContent)) {
-                $summary = trim($exNodes->item(0)->textContent);
-                $description[] = mb_strimwidth($summary, 0, 101, '…'); // 100 + '…'(1)
-            }
-
-            $metadata->description = implode(' / ', $description);
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        // 作者名
+        $writerNodes = $xpath->query('//*[contains(@class, "novel_writername")]');
+        if ($writerNodes->length !== 0 && !empty($writerNodes->item(0)->textContent)) {
+            $description[] = trim($writerNodes->item(0)->textContent);
         }
+
+        // あらすじ
+        $exNodes = $xpath->query('//*[@id="novel_ex"]');
+        if ($exNodes->length !== 0 && !empty($exNodes->item(0)->textContent)) {
+            $summary = trim($exNodes->item(0)->textContent);
+            $description[] = mb_strimwidth($summary, 0, 101, '…'); // 100 + '…'(1)
+        }
+
+        $metadata->description = implode(' / ', $description);
+
+        return $metadata;
     }
 }

app/MetadataResolver/NicoSeigaResolver.php

@@ -3,6 +3,7 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
+use Symfony\Component\DomCrawler\Crawler;
 
 class NicoSeigaResolver implements Resolver
 {
@@ -24,16 +25,18 @@ class NicoSeigaResolver implements Resolver
     public function resolve(string $url): Metadata
     {
         $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $html = (string)$res->getBody();
+        $metadata = $this->ogpResolver->parse($html);
+        $crawler = new Crawler($html);
 
-            // ページURLからサムネイルURLに変換
-            preg_match('~http://(?:(?:sp\\.)?seiga\\.nicovideo\\.jp/seiga(?:/#!)?|nico\\.ms)/im(\\d+)~', $url, $matches);
-            $metadata->image = "http://lohas.nicoseiga.jp/thumb/${matches[1]}l?";
+        // タグ
+        $excludeTags = ['R-15'];
+        $metadata->tags = array_values(array_diff($crawler->filter('.tag')->extract(['_text']), $excludeTags));
 
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
-        }
+        // ページURLからサムネイルURLに変換
+        preg_match('~https?://(?:(?:sp\\.)?seiga\\.nicovideo\\.jp/seiga(?:/#!)?|nico\\.ms)/im(\\d+)~', $url, $matches);
+        $metadata->image = "https://lohas.nicoseiga.jp/thumb/${matches[1]}l?";
+
+        return $metadata;
     }
 }

app/MetadataResolver/NijieResolver.php

@@ -3,6 +3,8 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
+use Illuminate\Support\Str;
+use Symfony\Component\DomCrawler\Crawler;
 
 class NijieResolver implements Resolver
 {
@@ -30,27 +32,33 @@ class NijieResolver implements Resolver
             $url = preg_replace('~view_popup\.php~', 'view.php', $url);
         }
 
-        $client = $this->client;
-        $res = $client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $res =  $this->client->get($url);
+        $html = (string) $res->getBody();
+        $metadata = $this->ogpResolver->parse($html);
+        $crawler = new Crawler($html);
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
-            $dataNode = $xpath->query('//script[substring(@type, string-length(@type) - 3, 4) = "json"]');
-            foreach ($dataNode as $node) {
-                // 改行がそのまま入っていることがあるのでデコード前にエスケープが必要
-                $imageData = json_decode(preg_replace('/\r?\n/', '\n', $node->nodeValue), true);
-                if (isset($imageData['thumbnailUrl']) && !ends_with($imageData['thumbnailUrl'], '.gif') && !ends_with($imageData['thumbnailUrl'], '.mp4')) {
-                    $metadata->image = preg_replace('~nijie\\.info/.*/nijie_picture/~', 'nijie.info/nijie_picture/', $imageData['thumbnailUrl']);
-                    break;
-                }
-            }
+        $json = $crawler->filter('script[type="application/ld+json"]')->first()->text();
 
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        // 改行がそのまま入っていることがあるのでデコード前にエスケープが必要
+        $data = json_decode(preg_replace('/\r?\n/', '\n', $json), true);
+
+        // DomCrawler内でjson内の日本語がHTMLエンティティに変換されるので、全要素に対してhtml_entity_decode
+        array_walk_recursive($data, function (&$v) {
+            $v = html_entity_decode($v);
+        });
+
+        $metadata->title = $data['name'];
+        $metadata->description = '投稿者: ' . $data['author']['name'] . PHP_EOL . $data['description'];
+        if (
+            isset($data['thumbnailUrl']) &&
+            !Str::endsWith($data['thumbnailUrl'], '.gif') &&
+            !Str::endsWith($data['thumbnailUrl'], '.mp4')
+        ) {
+            // サムネイルからメイン画像に
+            $metadata->image = str_replace('__rs_l160x160/', '', $data['thumbnailUrl']);
         }
+        $metadata->tags = $crawler->filter('#view-tag span.tag_name')->extract('_text');
+
+        return $metadata;
     }
 }

app/MetadataResolver/OGPResolver.php

@@ -3,6 +3,8 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
+use GuzzleHttp\Cookie\CookieJar;
+use GuzzleHttp\RequestOptions;
 
 class OGPResolver implements Resolver, Parser
 {
@@ -18,12 +20,7 @@ class OGPResolver implements Resolver, Parser
 
     public function resolve(string $url): Metadata
     {
-        $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            return $this->parse($res->getBody());
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
-        }
+        return $this->parse($this->client->get($url, [RequestOptions::COOKIES => new CookieJar()])->getBody());
     }
 
     public function parse(string $html): Metadata

app/MetadataResolver/PatreonResolver.php

@@ -25,18 +25,14 @@ class PatreonResolver implements Resolver
     public function resolve(string $url): Metadata
     {
         $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $metadata = $this->ogpResolver->parse($res->getBody());
 
-            parse_str(parse_url($metadata->image, PHP_URL_QUERY), $query);
-            if (isset($query['token-time'])) {
-                $expires_at_unixtime = $query['token-time'];
-                $metadata->expires_at = Carbon::createFromTimestamp($expires_at_unixtime);
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        parse_str(parse_url($metadata->image, PHP_URL_QUERY), $query);
+        if (isset($query['token-time'])) {
+            $expires_at_unixtime = $query['token-time'];
+            $metadata->expires_at = Carbon::createFromTimestamp($expires_at_unixtime);
         }
+
+        return $metadata;
     }
 }

app/MetadataResolver/PixivResolver.php

@@ -38,52 +38,48 @@ class PixivResolver implements Resolver
     {
         if (preg_match('~www\.pixiv\.net/user/\d+/series/\d+~', $url, $matches)) {
             $res = $this->client->get($url);
-            if ($res->getStatusCode() === 200) {
-                $metadata = $this->ogpResolver->parse($res->getBody());
-                $metadata->image = $this->proxize($metadata->image);
+            $metadata = $this->ogpResolver->parse($res->getBody());
+            $metadata->image = $this->proxize($metadata->image);
 
-                return $metadata;
-            } else {
-                throw new \RuntimeException("{$res->getStatusCode()}: $url");
-            }
+            return $metadata;
         }
 
-        parse_str(parse_url($url, PHP_URL_QUERY), $params);
-        $illustId = $params['illust_id'];
         $page = 0;
+        if (preg_match('~www\.pixiv\.net/(en/)?artworks/(?P<illustId>\d+)~', $url, $matches)) {
+            $illustId = $matches['illustId'];
+        } else {
+            parse_str(parse_url($url, PHP_URL_QUERY), $params);
+            $illustId = $params['illust_id'];
 
-        // 漫画ページ（ページ数はmanga_bigならあるかも）
-        if ($params['mode'] === 'manga_big' || $params['mode'] === 'manga') {
-            $page = $params['page'] ?? 0;
+            // 漫画ページ（ページ数はmanga_bigならあるかも）
+            if ($params['mode'] === 'manga_big' || $params['mode'] === 'manga') {
+                $page = $params['page'] ?? 0;
+            }
         }
 
         $res = $this->client->get('https://www.pixiv.net/ajax/illust/' . $illustId);
-        if ($res->getStatusCode() === 200) {
-            $json = json_decode($res->getBody()->getContents(), true);
-            $metadata = new Metadata();
+        $json = json_decode($res->getBody()->getContents(), true);
+        $metadata = new Metadata();
 
-            $metadata->title = $json['body']['illustTitle'] ?? '';
-            $metadata->description = '投稿者: ' . $json['body']['userName'] . PHP_EOL . strip_tags(str_replace('<br />', PHP_EOL, $json['body']['illustComment'] ?? ''));
-            $metadata->image = $this->proxize($json['body']['urls']['regular'] ?? '');
+        $metadata->title = $json['body']['illustTitle'] ?? '';
+        $metadata->description = '投稿者: ' . $json['body']['userName'] . PHP_EOL . strip_tags(str_replace('<br />', PHP_EOL, $json['body']['illustComment'] ?? ''));
+        $metadata->image = $this->proxize($json['body']['urls']['regular'] ?? '');
 
-            // ページ数の指定がある場合は画像URLをそのページにする
-            if ($page != 0) {
-                $metadata->image = str_replace('_p0', '_p'.$page, $metadata->image);
-            }
+        // ページ数の指定がある場合は画像URLをそのページにする
+        if ($page != 0) {
+            $metadata->image = str_replace('_p0', '_p' . $page, $metadata->image);
+        }
 
-            // タグ
-            if (!empty($json['body']['tags']['tags'])) {
-                foreach ($json['body']['tags']['tags'] as $tag) {
-                    // 一部の固定キーワードは無視
-                    if (array_search($tag['tag'], ['R-18', 'イラスト', 'pixiv', 'ピクシブ'], true) === false) {
-                        $metadata->tags[] = preg_replace('/\s/', '_', $tag['tag']);
-                    }
+        // タグ
+        if (!empty($json['body']['tags']['tags'])) {
+            foreach ($json['body']['tags']['tags'] as $tag) {
+                // 一部の固定キーワードは無視
+                if (array_search($tag['tag'], ['R-18', 'イラスト', 'pixiv', 'ピクシブ'], true) === false) {
+                    $metadata->tags[] = preg_replace('/\s/', '_', $tag['tag']);
                 }
             }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
         }
+
+        return $metadata;
     }
 }

app/MetadataResolver/PlurkResolver.php

@@ -24,21 +24,17 @@ class PlurkResolver implements Resolver
     public function resolve(string $url): Metadata
     {
         $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $metadata = $this->ogpResolver->parse($res->getBody());
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
-            $imageNode = $xpath->query('//div[@class="text_holder"]/a[1]')->item(0);
+        $dom = new \DOMDocument();
+        @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
+        $xpath = new \DOMXPath($dom);
+        $imageNode = $xpath->query('//div[@class="text_holder"]/a[1]')->item(0);
 
-            if ($imageNode) {
-                $metadata->image = $imageNode->getAttribute('href');
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        if ($imageNode) {
+            $metadata->image = $imageNode->getAttribute('href');
         }
+
+        return $metadata;
     }
 }

app/MetadataResolver/ResolverCircuitBreakException.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use Throwable;
+
+/**
+ * 規定回数以上の解決失敗により、メタデータの取得が不能となっている場合にスローされます。
+ */
+class ResolverCircuitBreakException extends \RuntimeException
+{
+    public function __construct(int $errorCount, string $url, Throwable $previous = null)
+    {
+        parent::__construct("{$errorCount}回失敗しているためメタデータの取得を中断しました: {$url}", 0, $previous);
+    }
+}

app/MetadataResolver/SteamResolver.php

@@ -24,21 +24,17 @@ class SteamResolver implements Resolver
         $appid = $matches[1];
 
         $res = $this->client->get('https://store.steampowered.com/api/appdetails/?l=japanese&appids=' . $appid);
-        if ($res->getStatusCode() === 200) {
-            $json = json_decode($res->getBody()->getContents(), true);
-            if ($json[$appid]['success'] === false) {
-                throw new \RuntimeException("API response [$appid][success] is false: $url");
-            }
-            $data = $json[$appid]['data'];
-            $metadata = new Metadata();
-
-            $metadata->title = $data['name'] ?? '';
-            $metadata->description = strip_tags(str_replace('<br />', PHP_EOL, html_entity_decode($data['short_description'] ?? '')));
-            $metadata->image = $data['header_image'] ?? '';
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        $json = json_decode($res->getBody()->getContents(), true);
+        if ($json[$appid]['success'] === false) {
+            throw new \RuntimeException("API response [$appid][success] is false: $url");
         }
+        $data = $json[$appid]['data'];
+        $metadata = new Metadata();
+
+        $metadata->title = $data['name'] ?? '';
+        $metadata->description = strip_tags(str_replace('<br />', PHP_EOL, html_entity_decode($data['short_description'] ?? '')));
+        $metadata->image = $data['header_image'] ?? '';
+
+        return $metadata;
     }
 }

app/MetadataResolver/ToranoanaResolver.php

@@ -25,20 +25,16 @@ class ToranoanaResolver implements Resolver
     public function resolve(string $url): Metadata
     {
         $res = $this->client->get($url);
-        if ($res->getStatusCode() === 200) {
-            $metadata = $this->ogpResolver->parse($res->getBody());
+        $metadata = $this->ogpResolver->parse($res->getBody());
 
-            $dom = new \DOMDocument();
-            @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
-            $xpath = new \DOMXPath($dom);
-            $imgNode = $xpath->query('//*[@id="preview"]//img')->item(0);
-            if ($imgNode !== null) {
-                $metadata->image = $imgNode->getAttribute('src');
-            }
-
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
+        $dom = new \DOMDocument();
+        @$dom->loadHTML(mb_convert_encoding($res->getBody(), 'HTML-ENTITIES', 'UTF-8'));
+        $xpath = new \DOMXPath($dom);
+        $imgNode = $xpath->query('//*[@id="preview"]//img')->item(0);
+        if ($imgNode !== null) {
+            $metadata->image = $imgNode->getAttribute('src');
         }
+
+        return $metadata;
     }
 }

app/MetadataResolver/TwitterResolver.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use GuzzleHttp\Client;
+
+class TwitterResolver implements Resolver
+{
+    /**
+     * @var Client
+     */
+    private $client;
+    /**
+     * @var OGPResolver
+     */
+    private $ogpResolver;
+
+    public function __construct(Client $client, OGPResolver $ogpResolver)
+    {
+        $this->client = $client;
+        $this->ogpResolver = $ogpResolver;
+    }
+
+    public function resolve(string $url): Metadata
+    {
+        $url = preg_replace('/(www\.)?(mobile|m)\.twitter\.com/u', 'twitter.com', $url);
+
+        $res = $this->client->get($url);
+        $html = (string) $res->getBody();
+
+        return $this->ogpResolver->parse($html);
+    }
+}

app/MetadataResolver/UncaughtResolverException.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace App\MetadataResolver;
+
+/**
+ * MetadataResolver内で未キャッチの例外が発生した場合にスローされます。
+ */
+class UncaughtResolverException extends \RuntimeException
+{
+}

app/MetadataResolver/UnsupportedContentException.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace App\MetadataResolver;
+
+use Exception;
+
+/**
+ * このResolverやParserが対応していないサイトであったことを表わします。
+ */
+class UnsupportedContentException extends Exception
+{
+}

app/MetadataResolver/XtubeResolver.php

@@ -3,6 +3,7 @@
 namespace App\MetadataResolver;
 
 use GuzzleHttp\Client;
+use Symfony\Component\DomCrawler\Crawler;
 
 class XtubeResolver implements Resolver
 {
@@ -10,32 +11,34 @@ class XtubeResolver implements Resolver
      * @var Client
      */
     private $client;
+    /**
+     * @var OGPResolver
+     */
+    private $ogpResolver;
 
-    public function __construct(Client $client)
+    public function __construct(Client $client, OGPResolver $ogpResolver)
     {
         $this->client = $client;
+        $this->ogpResolver = $ogpResolver;
     }
 
     public function resolve(string $url): Metadata
     {
-        if (preg_match('~www\.xtube\.com/video-watch/.*-(\d+)$~', $url, $matches) !== 1) {
+        if (preg_match('~www\.xtube\.com/video-watch/.*-(\d+)$~', $url) !== 1) {
             throw new \RuntimeException("Unmatched URL Pattern: $url");
         }
-        $videoid = $matches[1];
 
-        $res = $this->client->get('https://www.xtube.com/webmaster/api/getvideobyid?video_id=' . $videoid);
-        if ($res->getStatusCode() === 200) {
-            $data = json_decode($res->getBody()->getContents(), true);
-            $metadata = new Metadata();
+        $res = $this->client->get($url);
+        $html = (string) $res->getBody();
+        $metadata = $this->ogpResolver->parse($html);
+        $crawler = new Crawler($html);
 
-            $metadata->title = $data['title'] ?? '';
-            $metadata->description = strip_tags(str_replace('\n', PHP_EOL, html_entity_decode($data['description'] ?? '')));
-            $metadata->image = str_replace('eSuQ8f', 'eSK08f', $data['thumb'] ?? ''); // 300x169 to 300x210
-            $metadata->tags = array_values(array_unique($data['tags']));
+        $metadata->title = trim($crawler->filter('.underPlayerRateForm h1')->text(''));
+        // $metadata->description =  trim($crawler->filter('.fullDescription ')->text(''));
+        $metadata->image = str_replace('m=eSuQ8f', 'm=eaAaaEFb', $metadata->image);
+        $metadata->image = str_replace('240X180', 'original', $metadata->image);
+        $metadata->tags = array_map('trim', $crawler->filter('.tagsCategories a')->extract('_text'));
 
-            return $metadata;
-        } else {
-            throw new \RuntimeException("{$res->getStatusCode()}: $url");
-        }
+        return $metadata;
     }
 }

app/Policies/EjaculationPolicy.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Policies;
+
+use App\Ejaculation;
+use App\User;
+use Illuminate\Auth\Access\HandlesAuthorization;
+
+class EjaculationPolicy
+{
+    use HandlesAuthorization;
+
+    /**
+     * Create a new policy instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    public function edit(User $user, Ejaculation $ejaculation): bool
+    {
+        return $user->id === $ejaculation->user_id;
+    }
+}

app/Providers/AppServiceProvider.php

@@ -3,6 +3,8 @@
 namespace App\Providers;
 
 use App\MetadataResolver\MetadataResolver;
+use GuzzleHttp\Client;
+use GuzzleHttp\RequestOptions;
 use Illuminate\Support\Facades\Blade;
 use Illuminate\Support\ServiceProvider;
 use Parsedown;
@@ -19,6 +21,8 @@ class AppServiceProvider extends ServiceProvider
         Blade::directive('parsedown', function ($expression) {
             return "<?php echo app('parsedown')->text($expression); ?>";
         });
+
+        stream_filter_register('convert.mbstring.*', 'Stream_Filter_Mbstring');
     }
 
     /**
@@ -34,5 +38,12 @@ class AppServiceProvider extends ServiceProvider
         $this->app->singleton('parsedown', function () {
             return Parsedown::instance();
         });
+        $this->app->bind(Client::class, function () {
+            return new Client([
+                RequestOptions::HEADERS => [
+                    'User-Agent' => 'TissueBot/1.0'
+                ]
+            ]);
+        });
     }
 }

app/Providers/AuthServiceProvider.php

@@ -2,6 +2,8 @@
 
 namespace App\Providers;
 
+use App\Ejaculation;
+use App\Policies\EjaculationPolicy;
 use Illuminate\Foundation\Support\Providers\AuthServiceProvider as ServiceProvider;
 use Illuminate\Support\Facades\Gate;
 
@@ -14,6 +16,7 @@ class AuthServiceProvider extends ServiceProvider
      */
     protected $policies = [
         'App\Model' => 'App\Policies\ModelPolicy',
+        Ejaculation::class => EjaculationPolicy::class,
     ];
 
     /**

app/Providers/EventServiceProvider.php

@@ -2,6 +2,8 @@
 
 namespace App\Providers;
 
+use Illuminate\Auth\Events\Registered;
+use Illuminate\Auth\Listeners\SendEmailVerificationNotification;
 use Illuminate\Foundation\Support\Providers\EventServiceProvider as ServiceProvider;
 use Illuminate\Support\Facades\Event;
 
@@ -13,6 +15,9 @@ class EventServiceProvider extends ServiceProvider
      * @var array
      */
     protected $listen = [
+        Registered::class => [
+            SendEmailVerificationNotification::class,
+        ],
         'App\Events\LinkDiscovered' => [
             'App\Listeners\LinkCollector'
         ]

app/Rules/CsvDateTime.php

@@ -0,0 +1,82 @@
+<?php
+
+namespace App\Rules;
+
+use Illuminate\Contracts\Validation\Rule;
+
+/**
+ * CSVインポート機能の日時バリデーションルール
+ * @package App\Rules
+ */
+class CsvDateTime implements Rule
+{
+    const VALID_FORMATS = [
+        'Y/m/d H:i:s',
+        'Y/n/j G:i:s',
+        'Y/m/d H:i',
+        'Y/n/j G:i',
+    ];
+
+    const MINIMUM_TIMESTAMP = 946652400; // 2000-01-01 00:00:00 JST
+    const MAXIMUM_TIMESTAMP = 4102412399; // 2099-12-31 23:59:59 JST
+
+    /** @var string Validation error message */
+    private $message = ':attributeの形式は "年/月/日 時:分" にしてください。';
+
+    /**
+     * Create a new rule instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Determine if the validation rule passes.
+     *
+     * @param  string  $attribute
+     * @param  mixed  $value
+     * @return bool
+     */
+    public function passes($attribute, $value)
+    {
+        // この辺の実装の元ネタは、LaravelのValidatesAttributes#validateDateFormat()
+
+        if (!is_string($value)) {
+            return false;
+        }
+
+        foreach (self::VALID_FORMATS as $format) {
+            $date = \DateTime::createFromFormat('!' . $format, $value);
+            if (!$date) {
+                continue;
+            }
+
+            $timestamp = (int) $date->format('U');
+            if ($timestamp < self::MINIMUM_TIMESTAMP || self::MAXIMUM_TIMESTAMP < $timestamp) {
+                $this->message = ':attributeは 2000/01/01 00:00 〜 2099/12/31 23:59 の間のみ対応しています。';
+
+                return false;
+            }
+
+            $formatted = $date->format($format);
+            if ($formatted === $value) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Get the validation error message.
+     *
+     * @return string
+     */
+    public function message()
+    {
+        return $this->message;
+    }
+}

app/Rules/FuzzyBoolean.php

@@ -0,0 +1,62 @@
+<?php
+
+namespace App\Rules;
+
+use Illuminate\Contracts\Validation\Rule;
+
+class FuzzyBoolean implements Rule
+{
+    public static function isTruthy($value): bool
+    {
+        if ($value === 1 || $value === '1') {
+            return true;
+        }
+
+        $lower = strtolower((string)$value);
+
+        return $lower === 'true';
+    }
+
+    public static function isFalsy($value): bool
+    {
+        if ($value === null || $value === '' || $value === 0 || $value === '0') {
+            return true;
+        }
+
+        $lower = strtolower((string)$value);
+
+        return $lower === 'false';
+    }
+
+    /**
+     * Create a new rule instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        //
+    }
+
+    /**
+     * Determine if the validation rule passes.
+     *
+     * @param  string  $attribute
+     * @param  mixed  $value
+     * @return bool
+     */
+    public function passes($attribute, $value)
+    {
+        return self::isTruthy($value) || self::isFalsy($value);
+    }
+
+    /**
+     * Get the validation error message.
+     *
+     * @return string
+     */
+    public function message()
+    {
+        return __('validation.boolean');
+    }
+}

app/Services/CheckinCsvExporter.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Services;
+
+use App\User;
+use Illuminate\Support\Facades\DB;
+use League\Csv\Writer;
+
+class CheckinCsvExporter
+{
+    /** @var User Target user */
+    private $user;
+    /** @var string Output filename */
+    private $filename;
+    /** @var string Output charset */
+    private $charset;
+
+    public function __construct(User $user, string $filename, string $charset)
+    {
+        $this->user = $user;
+        $this->filename = $filename;
+        $this->charset = $charset;
+    }
+
+    public function execute()
+    {
+        $csv = Writer::createFromPath($this->filename, 'wb');
+        $csv->setNewline("\r\n");
+        if ($this->charset === 'SJIS-win') {
+            $csv->addStreamFilter('convert.mbstring.encoding.UTF-8:SJIS-win');
+        }
+
+        $header = ['日時', 'ノート', 'オカズリンク', '非公開', 'センシティブ'];
+        for ($i = 1; $i <= 32; $i++) {
+            $header[] = "タグ{$i}";
+        }
+        $csv->insertOne($header);
+
+        DB::transaction(function () use ($csv) {
+            // TODO: そんなに読み取り整合性を保つ努力はしていないのと、chunkの件数これでいいか分からない
+            $this->user->ejaculations()->with('tags')->orderBy('ejaculated_date')
+                ->chunk(1000, function ($ejaculations) use ($csv) {
+                    foreach ($ejaculations as $ejaculation) {
+                        $record = [
+                            $ejaculation->ejaculated_date->format('Y/m/d H:i'),
+                            $ejaculation->note,
+                            $ejaculation->link,
+                            self::formatBoolean($ejaculation->is_private),
+                            self::formatBoolean($ejaculation->is_too_sensitive),
+                        ];
+                        foreach ($ejaculation->tags->take(32) as $tag) {
+                            $record[] = $tag->name;
+                        }
+                        $csv->insertOne($record);
+                    }
+                });
+        });
+    }
+
+    private static function formatBoolean($value): string
+    {
+        return $value ? 'true' : 'false';
+    }
+}

app/Services/CheckinCsvImporter.php

@@ -0,0 +1,221 @@
+<?php
+declare(strict_types=1);
+
+namespace App\Services;
+
+use App\Ejaculation;
+use App\Exceptions\CsvImportException;
+use App\Rules\CsvDateTime;
+use App\Rules\FuzzyBoolean;
+use App\Tag;
+use App\User;
+use Carbon\Carbon;
+use Illuminate\Database\QueryException;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Validator;
+use League\Csv\Reader;
+use Throwable;
+
+class CheckinCsvImporter
+{
+    /** @var int 取り込み件数の上限 */
+    private const IMPORT_LIMIT = 5000;
+
+    /** @var User Target user */
+    private $user;
+    /** @var string CSV filename */
+    private $filename;
+
+    public function __construct(User $user, string $filename)
+    {
+        $this->user = $user;
+        $this->filename = $filename;
+    }
+
+    /**
+     * インポート処理を実行します。
+     * @return int 取り込んだ件数
+     */
+    public function execute(): int
+    {
+        // Guess charset
+        $charset = $this->guessCharset($this->filename);
+
+        // Open CSV
+        $csv = Reader::createFromPath($this->filename, 'r');
+        $csv->setHeaderOffset(0);
+        if ($charset === 'SJIS-win') {
+            $csv->addStreamFilter('convert.mbstring.encoding.SJIS-win:UTF-8');
+        }
+
+        // Import
+        return DB::transaction(function () use ($csv) {
+            $alreadyImportedCount = $this->user->ejaculations()->where('ejaculations.source', Ejaculation::SOURCE_CSV)->count();
+            $errors = [];
+
+            if (!in_array('日時', $csv->getHeader(), true)) {
+                $errors[] = '日時列は必須です。';
+            }
+
+            if (!empty($errors)) {
+                throw new CsvImportException(...$errors);
+            }
+
+            $imported = 0;
+            foreach ($csv->getRecords() as $offset => $record) {
+                $line = $offset + 1;
+                if (self::IMPORT_LIMIT <= $alreadyImportedCount + $imported) {
+                    $limit = self::IMPORT_LIMIT;
+                    $errors[] = "{$line} 行 : インポート機能で取り込めるデータは{$limit}件までに制限されています。これ以上取り込みできません。";
+                    throw new CsvImportException(...$errors);
+                }
+
+                $ejaculation = new Ejaculation(['user_id' => $this->user->id]);
+
+                $validator = Validator::make($record, [
+                    '日時' => ['required', new CsvDateTime()],
+                    'ノート' => 'nullable|string|max:500',
+                    'オカズリンク' => 'nullable|url|max:2000',
+                    '非公開' => ['nullable', new FuzzyBoolean()],
+                    'センシティブ' => ['nullable', new FuzzyBoolean()],
+                ]);
+
+                if ($validator->fails()) {
+                    foreach ($validator->errors()->all() as $message) {
+                        $errors[] = "{$line} 行 : {$message}";
+                    }
+                    continue;
+                }
+
+                $ejaculation->ejaculated_date = Carbon::createFromFormat('!Y/m/d H:i+', $record['日時']);
+                $ejaculation->note = str_replace(["\r\n", "\r"], "\n", $record['ノート'] ?? '');
+                $ejaculation->link = $record['オカズリンク'] ?? '';
+                $ejaculation->source = Ejaculation::SOURCE_CSV;
+                if (isset($record['非公開'])) {
+                    $ejaculation->is_private = FuzzyBoolean::isTruthy($record['非公開']);
+                }
+                if (isset($record['センシティブ'])) {
+                    $ejaculation->is_too_sensitive = FuzzyBoolean::isTruthy($record['センシティブ']);
+                }
+
+                try {
+                    $tags = $this->parseTags($line, $record);
+                } catch (CsvImportException $e) {
+                    $errors = array_merge($errors, $e->getErrors());
+                    continue;
+                }
+
+                DB::beginTransaction();
+                try {
+                    $ejaculation->save();
+                    if (!empty($tags)) {
+                        $ejaculation->tags()->sync(collect($tags)->pluck('id'));
+                    }
+                    DB::commit();
+                    $imported++;
+                } catch (QueryException $e) {
+                    DB::rollBack();
+                    if ($e->errorInfo[0] === '23505') {
+                        $errors[] = "{$line} 行 : すでにこの日時のチェックインデータが存在します。";
+                        continue;
+                    }
+                    throw $e;
+                } catch (Throwable $e) {
+                    DB::rollBack();
+                    throw $e;
+                }
+            }
+
+            if (!empty($errors)) {
+                throw new CsvImportException(...$errors);
+            }
+
+            return $imported;
+        });
+    }
+
+    /**
+     * 指定されたファイルを読み込み、文字コードの判定を行います。
+     * @param string $filename CSVファイル名
+     * @param int $samplingLength ファイルの先頭から何バイトを判定に使用するかを指定
+     * @return string 検出した文字コード (UTF-8, SJIS-win, ...)
+     * @throws CsvImportException ファイルの読み込みに失敗した、文字コードを判定できなかった、または非対応文字コードを検出した場合にスロー
+     */
+    private function guessCharset(string $filename, int $samplingLength = 1024): string
+    {
+        $fp = fopen($filename, 'rb');
+        if (!$fp) {
+            throw new CsvImportException('CSVファイルの読み込み中にエラーが発生しました。');
+        }
+
+        try {
+            $head = fread($fp, $samplingLength);
+            if ($head === false) {
+                throw new CsvImportException('CSVファイルの読み込み中にエラーが発生しました。');
+            }
+
+            for ($addition = 0; $addition < 4; $addition++) {
+                $charset = mb_detect_encoding($head, ['ASCII', 'UTF-8', 'SJIS-win'], true);
+                if ($charset) {
+                    if (array_search($charset, ['UTF-8', 'SJIS-win'], true) === false) {
+                        throw new CsvImportException('文字コード判定に失敗しました。UTF-8 (BOM無し) または Shift_JIS をお使いください。');
+                    } else {
+                        return $charset;
+                    }
+                }
+
+                // 1バイト追加で読み込んだら、文字境界に到達して上手く判定できるかもしれない
+                if (feof($fp)) {
+                    break;
+                }
+                $next = fread($fp, 1);
+                if ($next === false) {
+                    throw new CsvImportException('CSVファイルの読み込み中にエラーが発生しました。');
+                }
+                $head .= $next;
+            }
+
+            throw new CsvImportException('文字コード判定に失敗しました。UTF-8 (BOM無し) または Shift_JIS をお使いください。');
+        } finally {
+            fclose($fp);
+        }
+    }
+
+    /**
+     * タグ列をパースします。
+     * @param int $line 現在の行番号 (1 origin)
+     * @param array $record 対象行のデータ
+     * @return Tag[]
+     * @throws CsvImportException バリデーションエラーが発生した場合にスロー
+     */
+    private function parseTags(int $line, array $record): array
+    {
+        $tags = [];
+        foreach (array_keys($record) as $column) {
+            if (preg_match('/\Aタグ\d{1,2}\z/u', $column) !== 1) {
+                continue;
+            }
+
+            $tag = trim($record[$column]);
+            if (empty($tag)) {
+                continue;
+            }
+            if (mb_strlen($tag) > 255) {
+                throw new CsvImportException("{$line} 行 : {$column}は255文字以内にしてください。");
+            }
+            if (strpos($tag, "\n") !== false) {
+                throw new CsvImportException("{$line} 行 : {$column}に改行を含めることはできません。");
+            }
+            if (strpos($tag, ' ') !== false) {
+                throw new CsvImportException("{$line} 行 : {$column}にスペースを含めることはできません。");
+            }
+
+            $tags[] = Tag::firstOrCreate(['name' => $tag]);
+            if (count($tags) >= 32) {
+                break;
+            }
+        }
+
+        return $tags;
+    }
+}

app/Services/MetadataResolveService.php

@@ -0,0 +1,306 @@
+<?php
+
+namespace App\Services;
+
+use App\ContentProvider;
+use App\Metadata;
+use App\MetadataResolver\DeniedHostException;
+use App\MetadataResolver\DisallowedByProviderException;
+use App\MetadataResolver\MetadataResolver;
+use App\MetadataResolver\ResolverCircuitBreakException;
+use App\MetadataResolver\UncaughtResolverException;
+use App\Tag;
+use App\Utilities\Formatter;
+use Carbon\Carbon;
+use Carbon\CarbonInterface;
+use GuzzleHttp\Client;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+
+class MetadataResolveService
+{
+    /** @var int メタデータの解決を中断するエラー回数。この回数以上エラーしていたら処理は行わない。 */
+    const CIRCUIT_BREAK_COUNT = 5;
+
+    /** @var MetadataResolver */
+    private $resolver;
+    /** @var Formatter */
+    private $formatter;
+
+    public function __construct(MetadataResolver $resolver, Formatter $formatter)
+    {
+        $this->resolver = $resolver;
+        $this->formatter = $formatter;
+    }
+
+    /**
+     * メタデータをキャッシュまたはリモートに問い合わせて取得します。
+     * @param string $url メタデータを取得したいURL
+     * @return Metadata 取得できたメタデータ
+     * @throws DeniedHostException アクセス先がブラックリスト入りしているため取得できなかった場合にスロー
+     * @throws UncaughtResolverException Resolver内で例外が発生して取得できなかった場合にスロー
+     */
+    public function execute(string $url): Metadata
+    {
+        // URLの正規化
+        $url = $this->formatter->normalizeUrl($url);
+
+        // 自分自身は解決しない
+        if (parse_url($url, PHP_URL_HOST) === parse_url(config('app.url'), PHP_URL_HOST)) {
+            throw new DeniedHostException($url);
+        }
+
+        $metadata = Metadata::find($url);
+
+        // 無かったら取得
+        // TODO: ある程度古かったら再取得とかありだと思う
+        if ($metadata == null || $metadata->needRefresh()) {
+            $hostWithPort = $this->getHostWithPortFromUrl($url);
+            $metadata = $this->hostLock($hostWithPort, function (?CarbonInterface $lastAccess) use ($url) {
+                // HostLockの解放待ちをしている間に、他のプロセスで取得完了しているかもしれない
+                $metadata = Metadata::find($url);
+                if ($metadata !== null && !$metadata->needRefresh()) {
+                    return $metadata;
+                }
+
+                $this->checkProviderPolicy($url, $lastAccess);
+
+                return $this->resolve($url, $metadata);
+            });
+        }
+
+        return $metadata;
+    }
+
+    /**
+     * URLからホスト部とポート部を抽出
+     * @param string $url
+     * @return string
+     */
+    private function getHostWithPortFromUrl(string $url): string
+    {
+        $parts = parse_url($url);
+        $host = $parts['host'];
+        if (isset($parts['port'])) {
+            $host .= ':' . $parts['port'];
+        }
+
+        return $host;
+    }
+
+    /**
+     * アクセス先ホスト単位の排他ロックを取って処理を実行
+     * @param string $host
+     * @param callable $fn
+     * @return mixed return of $fn
+     * @throws \RuntimeException いろいろな死に方をする
+     */
+    private function hostLock(string $host, callable $fn)
+    {
+        $lockDir = storage_path('content_providers_lock');
+        if (!file_exists($lockDir)) {
+            if (!mkdir($lockDir)) {
+                throw new \RuntimeException("Lock failed! Can't create lock directory.");
+            }
+        }
+
+        $lockFile = $lockDir . DIRECTORY_SEPARATOR . $host;
+        $fp = fopen($lockFile, 'c+b');
+        if ($fp === false) {
+            throw new \RuntimeException("Lock failed! Can't open lock file.");
+        }
+
+        try {
+            if (!flock($fp, LOCK_EX)) {
+                throw new \RuntimeException("Lock failed! Can't lock file.");
+            }
+
+            try {
+                $accessInfoText = stream_get_contents($fp);
+                if ($accessInfoText !== false) {
+                    $accessInfo = json_decode($accessInfoText, true);
+                }
+
+                $result = $fn(isset($accessInfo['time']) ? new Carbon($accessInfo['time']) : null);
+
+                $accessInfo = [
+                    'time' => now()->toIso8601String()
+                ];
+                fseek($fp, 0);
+                if (fwrite($fp, json_encode($accessInfo)) === false) {
+                    throw new \RuntimeException("I/O Error! Can't write to lock file.");
+                }
+
+                return $result;
+            } finally {
+                if (!flock($fp, LOCK_UN)) {
+                    throw new \RuntimeException("Unlock failed! Can't unlock file.");
+                }
+            }
+        } finally {
+            if (!fclose($fp)) {
+                throw new \RuntimeException("Unlock failed! Can't close lock file.");
+            }
+        }
+    }
+
+    /**
+     * 指定したメタデータURLのホストが持つrobots.txtをダウンロードします。
+     * @param string $url メタデータのURL
+     * @return string
+     */
+    private function fetchRobotsTxt(string $url): ?string
+    {
+        $parts = parse_url($url);
+        $robotsUrl = http_build_url([
+            'scheme' => $parts['scheme'],
+            'host' => $parts['host'],
+            'port' => $parts['port'] ?? null,
+            'path' => '/robots.txt'
+        ]);
+
+        $client = app(Client::class);
+        try {
+            $res = $client->get($robotsUrl);
+            if (stripos($res->getHeaderLine('Content-Type'), 'text/plain') !== 0) {
+                Log::error('robots.txtの取得に失敗: 不適切なContent-Type (' . $res->getHeaderLine('Content-Type') . ')');
+
+                return null;
+            }
+
+            return (string) $res->getBody();
+        } catch (\Exception $e) {
+            Log::error("robots.txtの取得に失敗: {$e}");
+
+            return null;
+        }
+    }
+
+    /**
+     * ContentProviderポリシー情報との照合を行い、アクセス可能かチェックします。アクセスできない場合は例外をスローします。
+     * @param string $url メタデータを取得したいURL
+     * @param CarbonInterface|null $lastAccess アクセス先ホストへの最終アクセス日時 (記録がある場合)
+     * @throws DeniedHostException アクセス先がTissue内のブラックリストに入っている場合にスロー
+     * @throws DisallowedByProviderException アクセス先のrobots.txtによって拒否されている場合にスロー
+     */
+    private function checkProviderPolicy(string $url, ?CarbonInterface $lastAccess): void
+    {
+        DB::beginTransaction();
+        try {
+            $hostWithPort = $this->getHostWithPortFromUrl($url);
+            $contentProvider = ContentProvider::sharedLock()->find($hostWithPort);
+            if ($contentProvider === null) {
+                $contentProvider = ContentProvider::create([
+                    'host' => $hostWithPort,
+                    'robots' => $this->fetchRobotsTxt($url),
+                    'robots_cached_at' => now(),
+                ]);
+            }
+
+            if ($contentProvider->is_blocked) {
+                throw new DeniedHostException($url);
+            }
+
+            // 連続アクセス制限
+            if ($lastAccess !== null) {
+                $elapsedSeconds = $lastAccess->diffInSeconds(now(), false);
+                if ($elapsedSeconds < $contentProvider->access_interval_sec) {
+                    if ($elapsedSeconds < 0) {
+                        $wait = abs($elapsedSeconds) + $contentProvider->access_interval_sec;
+                    } else {
+                        $wait = $contentProvider->access_interval_sec - $elapsedSeconds;
+                    }
+                    sleep($wait);
+                }
+            }
+
+            // Fetch robots.txt
+            if ($contentProvider->robots_cached_at->diffInDays(now()) >= 7) {
+                $contentProvider->update([
+                    'robots' => $this->fetchRobotsTxt($url),
+                    'robots_cached_at' => now(),
+                ]);
+            }
+
+            // Check robots.txt
+            $robotsParser = new \RobotsTxtParser($contentProvider->robots);
+            $robotsParser->setUserAgent('TissueBot');
+            $robotsDelay = $robotsParser->getDelay();
+            if ($robotsDelay !== 0 && $robotsDelay >= $contentProvider->access_interval_sec) {
+                $contentProvider->access_interval_sec = (int) $robotsDelay;
+                $contentProvider->save();
+            }
+            if ($robotsParser->isDisallowed(parse_url($url, PHP_URL_PATH))) {
+                throw new DisallowedByProviderException($url);
+            }
+
+            DB::commit();
+        } catch (DeniedHostException | DisallowedByProviderException $e) {
+            // ContentProviderのデータ更新は行うため
+            DB::commit();
+            throw $e;
+        } catch (\Exception $e) {
+            DB::rollBack();
+            throw $e;
+        }
+    }
+
+    /**
+     * メタデータをリモートサーバに問い合わせて取得します。
+     * @param string $url メタデータを取得したいURL
+     * @param Metadata|null $metadata キャッシュ済のメタデータ (存在する場合)
+     * @return Metadata 取得できたメタデータ
+     * @throws UncaughtResolverException Resolver内で例外が発生して取得できなかった場合にスロー
+     * @throws ResolverCircuitBreakException 規定回数以上の解決失敗により、メタデータの取得が不能となっている場合にスロー
+     */
+    private function resolve(string $url, ?Metadata $metadata): Metadata
+    {
+        DB::beginTransaction();
+        try {
+            if ($metadata === null) {
+                $metadata = new Metadata(['url' => $url]);
+            }
+
+            if ($metadata->error_count >= self::CIRCUIT_BREAK_COUNT) {
+                throw new ResolverCircuitBreakException($metadata->error_count, $url);
+            }
+
+            try {
+                $resolved = $this->resolver->resolve($url);
+            } catch (\Exception $e) {
+                $metadata->storeException(now(), $e);
+                $metadata->save();
+                throw new UncaughtResolverException(implode(': ', [
+                    $metadata->error_count . '回目のメタデータ取得失敗', get_class($e), $e->getMessage()
+                ]), 0, $e);
+            }
+
+            $metadata->fill([
+                'title' => $resolved->title,
+                'description' => $resolved->description,
+                'image' => $resolved->image,
+                'expires_at' => $resolved->expires_at
+            ]);
+            $metadata->clearError();
+            $metadata->save();
+
+            $tagIds = [];
+            foreach ($resolved->normalizedTags() as $tagName) {
+                $tag = Tag::firstOrCreate(['name' => $tagName]);
+                $tagIds[] = $tag->id;
+            }
+            $metadata->tags()->sync($tagIds);
+
+            DB::commit();
+
+            return $metadata;
+        } catch (UncaughtResolverException $e) {
+            // Metadataにエラー情報を記録するため
+            DB::commit();
+            throw $e;
+        } catch (\Exception $e) {
+            DB::rollBack();
+            throw $e;
+        }
+    }
+}

app/Tag.php

@@ -2,6 +2,7 @@
 
 namespace App;
 
+use App\Utilities\Formatter;
 use Illuminate\Database\Eloquent\Model;
 
 class Tag extends Model
@@ -15,8 +16,22 @@ class Tag extends Model
         'name'
     ];
 
+    protected static function boot()
+    {
+        parent::boot();
+
+        self::creating(function (Tag $tag) {
+            $tag->normalized_name = app(Formatter::class)->normalizeTagName($tag->name);
+        });
+    }
+
     public function ejaculations()
     {
         return $this->belongsToMany('App\Ejaculation')->withTimestamps();
     }
+
+    public function metadata()
+    {
+        return $this->belongsToMany('App\Metadata')->withTimestamps();
+    }
 }

app/User.php

@@ -32,6 +32,15 @@ class User extends Authenticatable
         'password', 'remember_token',
     ];
 
+    /**
+     * The attributes that should be cast to native types.
+     *
+     * @var array
+     */
+    protected $casts = [
+         // 'email_verified_at' => 'datetime',
+    ];
+
     /**
      * このユーザのメールアドレスから、Gravatarの画像URLを生成します。
      * @param int $size 画像サイズ
@@ -53,8 +62,18 @@ class User extends Authenticatable
         return Auth::check() && $this->id === Auth::user()->id;
     }
 
+    public function ejaculations()
+    {
+        return $this->hasMany(Ejaculation::class);
+    }
+
     public function likes()
     {
         return $this->hasMany(Like::class);
     }
+
+    public function checkinWebhooks()
+    {
+        return $this->hasMany(CheckinWebhook::class);
+    }
 }

app/Utilities/Formatter.php

@@ -2,6 +2,7 @@
 
 namespace App\Utilities;
 
+use Illuminate\Support\Str;
 use Misd\Linkify\Linkify;
 
 class Formatter
@@ -55,10 +56,10 @@ class Formatter
         $parts = parse_url($url);
         if (!empty($parts['query'])) {
             // Remove query parameters
-            $url = str_replace_last('?' . $parts['query'], '', $url);
+            $url = Str::replaceFirst('?' . $parts['query'], '', $url);
             if (!empty($parts['fragment'])) {
                 // Remove fragment identifier
-                $url = str_replace_last('#' . $parts['fragment'], '', $url);
+                $url = Str::replaceFirst('#' . $parts['fragment'], '', $url);
             } else {
                 // "http://example.com/?query#" の場合 $parts['fragment'] は unset になるので、個別に判定して除去する必要がある
                 $url = preg_replace('/#\z/u', '', $url);
@@ -75,4 +76,68 @@ class Formatter
 
         return $url;
     }
+
+    /**
+     * imgタグのsrcsetで使用できる形式で、プロフィール画像URLを生成します。
+     * @param object $user Userなど、getProfileImageUrl()が実装されているオブジェクト
+     * @param int $baseSize 1x解像度における画像サイズ
+     * @param int $maxDensity 最高密度
+     * @return string srcset用の文字列
+     */
+    public function profileImageSrcSet($user, int $baseSize, int $maxDensity = 3)
+    {
+        $srcset = [];
+        for ($i = 1; $i <= $maxDensity; $i++) {
+            $srcset[] = $user->getProfileImageUrl($baseSize * $i) . " {$i}x";
+        }
+
+        return implode(',', $srcset);
+    }
+
+    /**
+     * php.ini書式のデータサイズを正規化します。
+     * @param mixed $val データサイズ
+     * @return string
+     */
+    public function normalizeIniBytes($val)
+    {
+        $val = trim($val);
+        $last = strtolower(substr($val, -1, 1));
+        if (ord($last) < 0x30 || ord($last) > 0x39) {
+            $bytes = substr($val, 0, -1);
+            switch ($last) {
+                case 'g':
+                    $bytes *= 1024;
+                    // fall through
+                    // no break
+                case 'm':
+                    $bytes *= 1024;
+                    // fall through
+                    // no break
+                case 'k':
+                    $bytes *= 1024;
+                    break;
+            }
+        } else {
+            $bytes = $val;
+        }
+
+        if ($bytes >= (1 << 30)) {
+            return ($bytes >> 30) . 'GB';
+        } elseif ($bytes >= (1 << 20)) {
+            return ($bytes >> 20) . 'MB';
+        } elseif ($bytes >= (1 << 10)) {
+            return ($bytes >> 10) . 'KB';
+        }
+
+        return $bytes . 'B';
+    }
+
+    public function normalizeTagName(string $name)
+    {
+        $name = \Normalizer::normalize($name, \Normalizer::FORM_KC);
+        $name = mb_strtolower($name);
+
+        return $name;
+    }
 }

bootstrap/app.php

@@ -12,7 +12,7 @@
 */
 
 $app = new Illuminate\Foundation\Application(
-    realpath(__DIR__.'/../')
+    $_ENV['APP_BASE_PATH'] ?? dirname(__DIR__)
 );
 
 /*

composer.json

@@ -4,27 +4,47 @@
     "keywords": ["framework", "laravel"],
     "license": "MIT",
     "type": "project",
+    "repositories": [
+        {
+            "type": "vcs",
+            "url": "https://github.com/xcezx/Stream_Filter_Mbstring"
+        }
+    ],
     "require": {
-        "php": ">=7.1.0",
+        "php": "^7.2",
+        "ext-dom": "*",
+        "ext-intl": "*",
+        "ext-json": "*",
+        "ext-libxml": "*",
+        "ext-mbstring": "*",
+        "ext-pdo": "*",
         "anhskohbo/no-captcha": "^3.0",
         "doctrine/dbal": "^2.9",
-        "fideloper/proxy": "~3.3",
+        "erusev/parsedown": "^1.7",
+        "fideloper/proxy": "^4.0",
         "guzzlehttp/guzzle": "^6.3",
-        "laravel/framework": "5.5.*",
-        "laravel/tinker": "~1.0",
+        "jakeasmith/http_build_url": "^1.0",
+        "laravel/framework": "^6.2",
+        "laravel/helpers": "^1.2",
+        "laravel/tinker": "^2.0",
+        "league/csv": "^9.5",
         "misd/linkify": "^1.1",
+        "openpear/stream_filter_mbstring": "dev-master",
+        "sentry/sentry-laravel": "1.8.0",
         "staudenmeir/eloquent-eager-limit": "^1.0",
         "symfony/css-selector": "^4.3",
-        "symfony/dom-crawler": "^4.3"
+        "symfony/dom-crawler": "^4.3",
+        "t1gor/robots-txt-parser": "^0.2.4"
     },
     "require-dev": {
         "barryvdh/laravel-debugbar": "^3.1",
         "barryvdh/laravel-ide-helper": "^2.5",
-        "filp/whoops": "~2.0",
+        "facade/ignition": "^1.4",
         "friendsofphp/php-cs-fixer": "^2.14",
-        "fzaninotto/faker": "~1.4",
-        "mockery/mockery": "~1.0",
-        "phpunit/phpunit": "~6.0",
+        "fzaninotto/faker": "^1.9.1",
+        "mockery/mockery": "^1.0",
+        "nunomaduro/collision": "^3.0",
+        "phpunit/phpunit": "^8.0",
         "symfony/thanks": "^1.0"
     },
     "autoload": {
@@ -45,11 +65,11 @@
             "@php -r \"file_exists('.env') || copy('.env.example', '.env');\""
         ],
         "post-create-project-cmd": [
-            "@php artisan key:generate"
+            "@php artisan key:generate --ansi"
         ],
         "post-autoload-dump": [
             "Illuminate\\Foundation\\ComposerScripts::postAutoloadDump",
-            "@php artisan package:discover"
+            "@php artisan package:discover --ansi"
         ],
         "fix": [
             "php-cs-fixer fix --config=.php_cs.dist"
@@ -62,5 +82,12 @@
         "preferred-install": "dist",
         "sort-packages": true,
         "optimize-autoloader": true
-    }
+    },
+    "extra": {
+        "laravel": {
+            "dont-discover": []
+        }
+    },
+    "minimum-stability": "dev",
+    "prefer-stable": true
 }

config/app.php

@@ -38,7 +38,7 @@ return [
     |
     */
 
-    'debug' => env('APP_DEBUG', false),
+    'debug' => (bool) env('APP_DEBUG', false),
 
     /*
     |--------------------------------------------------------------------------
@@ -53,6 +53,8 @@ return [
 
     'url' => env('APP_URL', 'http://localhost'),
 
+    'asset_url' => env('ASSET_URL', null),
+
     /*
     |--------------------------------------------------------------------------
     | Application Timezone
@@ -92,6 +94,19 @@ return [
 
     'fallback_locale' => 'en',
 
+    /*
+    |--------------------------------------------------------------------------
+    | Faker Locale
+    |--------------------------------------------------------------------------
+    |
+    | This locale will be used by the Faker PHP library when generating fake
+    | data for your database seeds. For example, this will be used to get
+    | localized telephone numbers, street address information and more.
+    |
+    */
+
+    'faker_locale' => 'en_US',
+
     /*
     |--------------------------------------------------------------------------
     | Encryption Key
@@ -107,23 +122,6 @@ return [
 
     'cipher' => 'AES-256-CBC',
 
-    /*
-    |--------------------------------------------------------------------------
-    | Logging Configuration
-    |--------------------------------------------------------------------------
-    |
-    | Here you may configure the log settings for your application. Out of
-    | the box, Laravel uses the Monolog PHP logging library. This gives
-    | you a variety of powerful log handlers / formatters to utilize.
-    |
-    | Available Settings: "single", "daily", "syslog", "errorlog"
-    |
-    */
-
-    'log' => env('APP_LOG', 'single'),
-
-    'log_level' => env('APP_LOG_LEVEL', 'debug'),
-
     /*
     |--------------------------------------------------------------------------
     | Autoloaded Service Providers
@@ -194,6 +192,7 @@ return [
     'aliases' => [
 
         'App' => Illuminate\Support\Facades\App::class,
+        'Arr' => Illuminate\Support\Arr::class,
         'Artisan' => Illuminate\Support\Facades\Artisan::class,
         'Auth' => Illuminate\Support\Facades\Auth::class,
         'Blade' => Illuminate\Support\Facades\Blade::class,
@@ -223,6 +222,7 @@ return [
         'Schema' => Illuminate\Support\Facades\Schema::class,
         'Session' => Illuminate\Support\Facades\Session::class,
         'Storage' => Illuminate\Support\Facades\Storage::class,
+        'Str' => Illuminate\Support\Str::class,
         'URL' => Illuminate\Support\Facades\URL::class,
         'Validator' => Illuminate\Support\Facades\Validator::class,
         'View' => Illuminate\Support\Facades\View::class,

config/auth.php

@@ -44,6 +44,7 @@ return [
         'api' => [
             'driver' => 'token',
             'provider' => 'users',
+            'hash' => false,
         ],
     ],
 
@@ -96,7 +97,21 @@ return [
             'provider' => 'users',
             'table' => 'password_resets',
             'expire' => 60,
+            'throttle' => 60,
         ],
     ],
 
+    /*
+    |--------------------------------------------------------------------------
+    | Password Confirmation Timeout
+    |--------------------------------------------------------------------------
+    |
+    | Here you may define the amount of seconds before a password confirmation
+    | times out and the user is prompted to re-enter their password via the
+    | confirmation screen. By default, the timeout lasts for three hours.
+    |
+    */
+
+    'password_timeout' => 10800,
+
 ];

config/broadcasting.php

@@ -36,7 +36,8 @@ return [
             'secret' => env('PUSHER_APP_SECRET'),
             'app_id' => env('PUSHER_APP_ID'),
             'options' => [
-                //
+                'cluster' => env('PUSHER_APP_CLUSTER'),
+                'useTLS' => true,
             ],
         ],
 

config/cache.php

@@ -1,5 +1,7 @@
 <?php
 
+use Illuminate\Support\Str;
+
 return [
 
     /*
@@ -11,7 +13,8 @@ return [
     | using this caching library. This connection is used when another is
     | not explicitly specified when executing a given caching function.
     |
-    | Supported: "apc", "array", "database", "file", "memcached", "redis"
+    | Supported: "apc", "array", "database", "file",
+    |            "memcached", "redis", "dynamodb"
     |
     */
 
@@ -57,7 +60,7 @@ return [
                 env('MEMCACHED_PASSWORD'),
             ],
             'options' => [
-                // Memcached::OPT_CONNECT_TIMEOUT  => 2000,
+                // Memcached::OPT_CONNECT_TIMEOUT => 2000,
             ],
             'servers' => [
                 [
@@ -70,7 +73,16 @@ return [
 
         'redis' => [
             'driver' => 'redis',
-            'connection' => 'default',
+            'connection' => 'cache',
+        ],
+
+        'dynamodb' => [
+            'driver' => 'dynamodb',
+            'key' => env('AWS_ACCESS_KEY_ID'),
+            'secret' => env('AWS_SECRET_ACCESS_KEY'),
+            'region' => env('AWS_DEFAULT_REGION', 'us-east-1'),
+            'table' => env('DYNAMODB_CACHE_TABLE', 'cache'),
+            'endpoint' => env('DYNAMODB_ENDPOINT'),
         ],
 
     ],
@@ -86,6 +98,6 @@ return [
     |
     */
 
-    'prefix' => 'laravel',
+    'prefix' => env('CACHE_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_').'_cache'),
 
 ];

config/database.php

@@ -1,5 +1,7 @@
 <?php
 
+use Illuminate\Support\Str;
+
 return [
 
     /*
@@ -35,12 +37,15 @@ return [
 
         'sqlite' => [
             'driver' => 'sqlite',
+            'url' => env('DATABASE_URL'),
             'database' => env('DB_DATABASE', database_path('database.sqlite')),
             'prefix' => '',
+            'foreign_key_constraints' => env('DB_FOREIGN_KEYS', true),
         ],
 
         'mysql' => [
             'driver' => 'mysql',
+            'url' => env('DATABASE_URL'),
             'host' => env('DB_HOST', '127.0.0.1'),
             'port' => env('DB_PORT', '3306'),
             'database' => env('DB_DATABASE', 'forge'),
@@ -50,12 +55,17 @@ return [
             'charset' => 'utf8mb4',
             'collation' => 'utf8mb4_unicode_ci',
             'prefix' => '',
+            'prefix_indexes' => true,
             'strict' => true,
             'engine' => null,
+            'options' => extension_loaded('pdo_mysql') ? array_filter([
+                PDO::MYSQL_ATTR_SSL_CA => env('MYSQL_ATTR_SSL_CA'),
+            ]) : [],
         ],
 
         'pgsql' => [
             'driver' => 'pgsql',
+            'url' => env('DATABASE_URL'),
             'host' => env('DB_HOST', '127.0.0.1'),
             'port' => env('DB_PORT', '5432'),
             'database' => env('DB_DATABASE', 'forge'),
@@ -63,12 +73,14 @@ return [
             'password' => env('DB_PASSWORD', ''),
             'charset' => 'utf8',
             'prefix' => '',
+            'prefix_indexes' => true,
             'schema' => 'public',
             'sslmode' => 'prefer',
         ],
 
         'sqlsrv' => [
             'driver' => 'sqlsrv',
+            'url' => env('DATABASE_URL'),
             'host' => env('DB_HOST', 'localhost'),
             'port' => env('DB_PORT', '1433'),
             'database' => env('DB_DATABASE', 'forge'),
@@ -76,6 +88,7 @@ return [
             'password' => env('DB_PASSWORD', ''),
             'charset' => 'utf8',
             'prefix' => '',
+            'prefix_indexes' => true,
         ],
 
     ],
@@ -99,20 +112,34 @@ return [
     |--------------------------------------------------------------------------
     |
     | Redis is an open source, fast, and advanced key-value store that also
-    | provides a richer set of commands than a typical key-value systems
+    | provides a richer body of commands than a typical key-value system
     | such as APC or Memcached. Laravel makes it easy to dig right in.
     |
     */
 
     'redis' => [
 
-        'client' => 'predis',
+        'client' => env('REDIS_CLIENT', 'phpredis'),
+
+        'options' => [
+            'cluster' => env('REDIS_CLUSTER', 'redis'),
+            'prefix' => env('REDIS_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_').'_database_'),
+        ],
 
         'default' => [
+            'url' => env('REDIS_URL'),
             'host' => env('REDIS_HOST', '127.0.0.1'),
             'password' => env('REDIS_PASSWORD', null),
-            'port' => env('REDIS_PORT', 6379),
-            'database' => 0,
+            'port' => env('REDIS_PORT', '6379'),
+            'database' => env('REDIS_DB', '0'),
+        ],
+
+        'cache' => [
+            'url' => env('REDIS_URL'),
+            'host' => env('REDIS_HOST', '127.0.0.1'),
+            'password' => env('REDIS_PASSWORD', null),
+            'port' => env('REDIS_PORT', '6379'),
+            'database' => env('REDIS_CACHE_DB', '1'),
         ],
 
     ],

config/filesystems.php

@@ -37,7 +37,7 @@ return [
     | may even configure multiple disks of the same driver. Defaults have
     | been setup for each driver as an example of the required options.
     |
-    | Supported Drivers: "local", "ftp", "s3", "rackspace"
+    | Supported Drivers: "local", "ftp", "sftp", "s3"
     |
     */
 
@@ -61,6 +61,8 @@ return [
             'secret' => env('AWS_SECRET'),
             'region' => env('AWS_REGION'),
             'bucket' => env('AWS_BUCKET'),
+            'url' => env('AWS_URL'),
+            'endpoint' => env('AWS_ENDPOINT'),
         ],
 
     ],

config/hashing.php

@@ -0,0 +1,52 @@
+<?php
+
+return [
+
+    /*
+    |--------------------------------------------------------------------------
+    | Default Hash Driver
+    |--------------------------------------------------------------------------
+    |
+    | This option controls the default hash driver that will be used to hash
+    | passwords for your application. By default, the bcrypt algorithm is
+    | used; however, you remain free to modify this option if you wish.
+    |
+    | Supported: "bcrypt", "argon", "argon2id"
+    |
+    */
+
+    'driver' => 'bcrypt',
+
+    /*
+    |--------------------------------------------------------------------------
+    | Bcrypt Options
+    |--------------------------------------------------------------------------
+    |
+    | Here you may specify the configuration options that should be used when
+    | passwords are hashed using the Bcrypt algorithm. This will allow you
+    | to control the amount of time it takes to hash the given password.
+    |
+    */
+
+    'bcrypt' => [
+        'rounds' => env('BCRYPT_ROUNDS', 10),
+    ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Argon Options
+    |--------------------------------------------------------------------------
+    |
+    | Here you may specify the configuration options that should be used when
+    | passwords are hashed using the Argon algorithm. These will allow you
+    | to control the amount of time it takes to hash the given password.
+    |
+    */
+
+    'argon' => [
+        'memory' => 1024,
+        'threads' => 2,
+        'time' => 2,
+    ],
+
+];

config/logging.php

@@ -0,0 +1,102 @@
+<?php
+
+use Monolog\Handler\StreamHandler;
+use Monolog\Handler\SyslogUdpHandler;
+
+return [
+
+    /*
+    |--------------------------------------------------------------------------
+    | Default Log Channel
+    |--------------------------------------------------------------------------
+    |
+    | This option defines the default log channel that gets used when writing
+    | messages to the logs. The name specified in this option should match
+    | one of the channels defined in the "channels" configuration array.
+    |
+    */
+
+    'default' => env('LOG_CHANNEL', 'stack'),
+
+    /*
+    |--------------------------------------------------------------------------
+    | Log Channels
+    |--------------------------------------------------------------------------
+    |
+    | Here you may configure the log channels for your application. Out of
+    | the box, Laravel uses the Monolog PHP logging library. This gives
+    | you a variety of powerful log handlers / formatters to utilize.
+    |
+    | Available Drivers: "single", "daily", "slack", "syslog",
+    |                    "errorlog", "monolog",
+    |                    "custom", "stack"
+    |
+    */
+
+    'channels' => [
+        'stack' => [
+            'driver' => 'stack',
+            'channels' => ['single'],
+        ],
+
+        'single' => [
+            'driver' => 'single',
+            'path' => storage_path('logs/laravel.log'),
+            'level' => 'debug',
+        ],
+
+        'daily' => [
+            'driver' => 'daily',
+            'path' => storage_path('logs/laravel.log'),
+            'level' => 'debug',
+            'days' => 14,
+        ],
+
+        'slack' => [
+            'driver' => 'slack',
+            'url' => env('LOG_SLACK_WEBHOOK_URL'),
+            'username' => 'Laravel Log',
+            'emoji' => ':boom:',
+            'level' => 'critical',
+        ],
+
+        'papertrail' => [
+            'driver' => 'monolog',
+            'level' => 'debug',
+            'handler' => SyslogUdpHandler::class,
+            'handler_with' => [
+                'host' => env('PAPERTRAIL_URL'),
+                'port' => env('PAPERTRAIL_PORT'),
+            ],
+        ],
+
+        'stderr' => [
+            'driver' => 'monolog',
+            'handler' => StreamHandler::class,
+            'formatter' => env('LOG_STDERR_FORMATTER'),
+            'with' => [
+                'stream' => 'php://stderr',
+            ],
+        ],
+
+        'syslog' => [
+            'driver' => 'syslog',
+            'level' => 'debug',
+        ],
+
+        'errorlog' => [
+            'driver' => 'errorlog',
+            'level' => 'debug',
+        ],
+
+        'null' => [
+            'driver' => 'monolog',
+            'handler' => NullHandler::class,
+        ],
+
+        'emergency' => [
+            'path' => storage_path('logs/laravel.log'),
+        ],
+    ],
+
+];

config/mail.php

@@ -11,8 +11,8 @@ return [
     | sending of e-mail. You may specify which one you're using throughout
     | your application here. By default, Laravel is setup for SMTP mail.
     |
-    | Supported: "smtp", "sendmail", "mailgun", "mandrill", "ses",
-    |            "sparkpost", "log", "array"
+    | Supported: "smtp", "sendmail", "mailgun", "ses",
+    |            "postmark", "log", "array"
     |
     */
 
@@ -120,4 +120,17 @@ return [
         ],
     ],
 
+    /*
+    |--------------------------------------------------------------------------
+    | Log Channel
+    |--------------------------------------------------------------------------
+    |
+    | If you are using the "log" driver, you may specify the logging channel
+    | if you prefer to keep mail messages separate from other log entries
+    | for simpler reading. Otherwise, the default channel will be used.
+    |
+    */
+
+    'log_channel' => env('MAIL_LOG_CHANNEL'),
+
 ];

config/queue.php

@@ -4,14 +4,12 @@ return [
 
     /*
     |--------------------------------------------------------------------------
-    | Default Queue Driver
+    | Default Queue Connection Name
     |--------------------------------------------------------------------------
     |
     | Laravel's queue API supports an assortment of back-ends via a single
     | API, giving you convenient access to each back-end using the same
-    | syntax for each one. Here you may set the default queue driver.
-    |
-    | Supported: "sync", "database", "beanstalkd", "sqs", "redis", "null"
+    | syntax for every one. Here you may define a default connection.
     |
     */
 
@@ -26,6 +24,8 @@ return [
     | is used by your application. A default configuration has been added
     | for each back-end shipped with Laravel. You are free to add more.
     |
+    | Drivers: "sync", "database", "beanstalkd", "sqs", "redis", "null"
+    |
     */
 
     'connections' => [
@@ -46,22 +46,24 @@ return [
             'host' => 'localhost',
             'queue' => 'default',
             'retry_after' => 90,
+            'block_for' => 0,
         ],
 
         'sqs' => [
             'driver' => 'sqs',
-            'key' => 'your-public-key',
-            'secret' => 'your-secret-key',
-            'prefix' => 'https://sqs.us-east-1.amazonaws.com/your-account-id',
-            'queue' => 'your-queue-name',
-            'region' => 'us-east-1',
+            'key' => env('AWS_ACCESS_KEY_ID'),
+            'secret' => env('AWS_SECRET_ACCESS_KEY'),
+            'prefix' => env('SQS_PREFIX', 'https://sqs.us-east-1.amazonaws.com/your-account-id'),
+            'queue' => env('SQS_QUEUE', 'your-queue-name'),
+            'region' => env('AWS_DEFAULT_REGION', 'us-east-1'),
         ],
 
         'redis' => [
             'driver' => 'redis',
             'connection' => 'default',
-            'queue' => 'default',
+            'queue' => env('REDIS_QUEUE', 'default'),
             'retry_after' => 90,
+            'block_for' => null,
         ],
 
     ],
@@ -78,6 +80,7 @@ return [
     */
 
     'failed' => [
+        'driver' => env('QUEUE_FAILED_DRIVER', 'database'),
         'database' => env('DB_CONNECTION', 'mysql'),
         'table' => 'failed_jobs',
     ],

config/sentry.php

@@ -0,0 +1,30 @@
+<?php
+
+return [
+
+    'dsn' => env('SENTRY_LARAVEL_DSN', env('SENTRY_DSN')),
+
+    // capture release as git sha
+    // 'release' => trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')),
+
+    'breadcrumbs' => [
+        // Capture Laravel logs in breadcrumbs
+        'logs' => true,
+
+        // Capture SQL queries in breadcrumbs
+        'sql_queries' => true,
+
+        // Capture bindings on SQL queries logged in breadcrumbs
+        'sql_bindings' => true,
+
+        // Capture queue job information in breadcrumbs
+        'queue_info' => true,
+
+        // Capture command information in breadcrumbs
+        'command_info' => true,
+    ],
+
+    // @see: https://docs.sentry.io/error-reporting/configuration/?platform=php#send-default-pii
+    'send_default_pii' => false,
+
+];

config/services.php

@@ -8,31 +8,26 @@ return [
     |--------------------------------------------------------------------------
     |
     | This file is for storing the credentials for third party services such
-    | as Stripe, Mailgun, SparkPost and others. This file provides a sane
-    | default location for this type of information, allowing packages
-    | to have a conventional place to find your various credentials.
+    | as Mailgun, SparkPost and others. This file provides a sane default
+    | location for this type of information, allowing packages to have
+    | a conventional file to locate the various service credentials.
     |
     */
 
     'mailgun' => [
         'domain' => env('MAILGUN_DOMAIN'),
         'secret' => env('MAILGUN_SECRET'),
+        'endpoint' => env('MAILGUN_ENDPOINT', 'api.mailgun.net'),
+    ],
+
+    'postmark' => [
+        'token' => env('POSTMARK_TOKEN'),
     ],
 
     'ses' => [
-        'key' => env('SES_KEY'),
-        'secret' => env('SES_SECRET'),
-        'region' => 'us-east-1',
-    ],
-
-    'sparkpost' => [
-        'secret' => env('SPARKPOST_SECRET'),
-    ],
-
-    'stripe' => [
-        'model' => App\User::class,
-        'key' => env('STRIPE_KEY'),
-        'secret' => env('STRIPE_SECRET'),
+        'key' => env('AWS_ACCESS_KEY_ID'),
+        'secret' => env('AWS_SECRET_ACCESS_KEY'),
+        'region' => env('AWS_DEFAULT_REGION', 'us-east-1'),
     ],
 
 ];

config/session.php

@@ -1,5 +1,7 @@
 <?php
 
+use Illuminate\Support\Str;
+
 return [
 
     /*
@@ -29,7 +31,7 @@ return [
     |
     */
 
-    'lifetime' => 120,
+    'lifetime' => env('SESSION_LIFETIME', 120),
 
     'expire_on_close' => false,
 
@@ -70,7 +72,7 @@ return [
     |
     */
 
-    'connection' => null,
+    'connection' => env('SESSION_CONNECTION', null),
 
     /*
     |--------------------------------------------------------------------------
@@ -96,7 +98,7 @@ return [
     |
     */
 
-    'store' => null,
+    'store' => env('SESSION_STORE', null),
 
     /*
     |--------------------------------------------------------------------------
@@ -122,7 +124,10 @@ return [
     |
     */
 
-    'cookie' => 'laravel_session',
+    'cookie' => env(
+        'SESSION_COOKIE',
+        Str::slug(env('APP_NAME', 'laravel'), '_').'_session'
+    ),
 
     /*
     |--------------------------------------------------------------------------
@@ -176,4 +181,19 @@ return [
 
     'http_only' => true,
 
+    /*
+    |--------------------------------------------------------------------------
+    | Same-Site Cookies
+    |--------------------------------------------------------------------------
+    |
+    | This option determines how your cookies behave when cross-site requests
+    | take place, and can be used to mitigate CSRF attacks. By default, we
+    | do not enable this as other CSRF protection services are in place.
+    |
+    | Supported: "lax", "strict", "none"
+    |
+    */
+
+    'same_site' => null,
+
 ];

config/view.php

@@ -28,6 +28,9 @@ return [
     |
     */
 
-    'compiled' => realpath(storage_path('framework/views')),
+    'compiled' => env(
+        'VIEW_COMPILED_PATH',
+        realpath(storage_path('framework/views'))
+    ),
 
 ];

database/.gitignore

@@ -1 +1,2 @@
 *.sqlite
+*.sqlite-journal