pepepper
/
tissue
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #465 from shikorism/feature/metadata-error 

メタデータ取得エラーの記録とリトライ制限の適用
This commit is contained in:
shibafu 2020-08-22 09:34:58 +09:00 committed by GitHub
parent 4e521baf56 ca070e773a
commit 3bb2b9afe0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 361 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Exceptions/Handler.php
View File

@ -23,6 +23,7 @@ class Handler extends ExceptionHandler
\Illuminate\Database\Eloquent\ModelNotFoundException::class,
\Illuminate\Session\TokenMismatchException::class,
\Illuminate\Validation\ValidationException::class,
\App\MetadataResolver\ResolverCircuitBreakException::class,
];
/**

60
app/Metadata.php
View File

@ -2,6 +2,8 @@
namespace App;
use Carbon\CarbonInterface;
use GuzzleHttp\Exception\RequestException;
use Illuminate\Database\Eloquent\Model;
class Metadata extends Model
@ -13,10 +15,66 @@ class Metadata extends Model
protected $fillable = ['url', 'title', 'description', 'image', 'expires_at'];
protected $visible = ['url', 'title', 'description', 'image', 'expires_at', 'tags'];
protected $dates = ['created_at', 'updated_at', 'expires_at'];
protected $dates = ['created_at', 'updated_at', 'expires_at', 'error_at'];
public function tags()
{
return $this->belongsToMany(Tag::class)->withTimestamps();
}
public function needRefresh(): bool
{
return $this->isExpired() || $this->error_at !== null;
}
public function isExpired(): bool
{
return $this->expires_at !== null && $this->expires_at < now();
}
public function storeException(CarbonInterface $error_at, \Exception $exception): self
{
$this->prepareFieldsOnError();
$this->error_at = $error_at;
$this->error_exception_class = get_class($exception);
$this->error_body = $exception->getMessage();
if ($exception instanceof RequestException) {
$this->error_http_code = $exception->getCode();
} else {
$this->error_http_code = null;
}
$this->error_count++;
return $this;
}
public function storeError(CarbonInterface $error_at, string $body, ?int $httpCode = null): self
{
$this->prepareFieldsOnError();
$this->error_at = $error_at;
$this->error_exception_class = null;
$this->error_body = $body;
$this->error_http_code = $httpCode;
$this->error_count++;
return $this;
}
public function clearError(): self
{
$this->error_at = null;
$this->error_exception_class = null;
$this->error_body = null;
$this->error_http_code = null;
$this->error_count = 0;
return $this;
}
private function prepareFieldsOnError()
{
$this->title = $this->title ?? '';
$this->description = $this->description ?? '';
$this->image = $this->image ?? '';
}
}

16
app/MetadataResolver/ResolverCircuitBreakException.php Normal file
View File

@ -0,0 +1,16 @@
<?php
namespace App\MetadataResolver;
use Throwable;
/**
* 規定回数以上の解決失敗により、メタデータの取得が不能となっている場合にスローされます。
*/
class ResolverCircuitBreakException extends \RuntimeException
{
public function __construct(int $errorCount, string $url, Throwable $previous = null)
{
parent::__construct("{$errorCount}回失敗しているためメタデータの取得を中断しました: {$url}", 0, $previous);
}
}

10
app/MetadataResolver/UncaughtResolverException.php Normal file
View File

@ -0,0 +1,10 @@
<?php
namespace App\MetadataResolver;
/**
* MetadataResolver内で未キャッチの例外が発生した場合にスローされます。
*/
class UncaughtResolverException extends \RuntimeException
{
}

80
app/Services/MetadataResolveService.php
View File

@ -5,14 +5,17 @@ namespace App\Services;
use App\Metadata;
use App\MetadataResolver\DeniedHostException;
use App\MetadataResolver\MetadataResolver;
use App\MetadataResolver\ResolverCircuitBreakException;
use App\MetadataResolver\UncaughtResolverException;
use App\Tag;
use App\Utilities\Formatter;
use GuzzleHttp\Exception\TransferException;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
class MetadataResolveService
{
/** @var int メタデータの解決を中断するエラー回数。この回数以上エラーしていたら処理は行わない。 */
const CIRCUIT_BREAK_COUNT = 5;
/** @var MetadataResolver */
private $resolver;
/** @var Formatter */
@ -24,6 +27,13 @@ class MetadataResolveService
$this->formatter = $formatter;
}
/**
* メタデータをキャッシュまたはリモートに問い合わせて取得します。
* @param string $url メタデータを取得したいURL
* @return Metadata 取得できたメタデータ
* @throws DeniedHostException アクセス先がブラックリスト入りしているため取得できなかった場合にスロー
* @throws UncaughtResolverException Resolver内で例外が発生して取得できなかった場合にスロー
*/
public function execute(string $url): Metadata
{
// URLの正規化
@ -34,34 +44,58 @@ class MetadataResolveService
throw new DeniedHostException($url);
}
return DB::transaction(function () use ($url) {
DB::beginTransaction();
try {
$metadata = Metadata::find($url);
// 無かったら取得
// TODO: ある程度古かったら再取得とかありだと思う
$metadata = Metadata::find($url);
if ($metadata == null || ($metadata->expires_at !== null && $metadata->expires_at < now())) {
if ($metadata == null || $metadata->needRefresh()) {
if ($metadata === null) {
$metadata = new Metadata(['url' => $url]);
}
if ($metadata->error_count >= self::CIRCUIT_BREAK_COUNT) {
throw new ResolverCircuitBreakException($metadata->error_count, $url);
}
try {
$resolved = $this->resolver->resolve($url);
$metadata = Metadata::updateOrCreate(['url' => $url], [
'title' => $resolved->title,
'description' => $resolved->description,
'image' => $resolved->image,
'expires_at' => $resolved->expires_at
]);
$tagIds = [];
foreach ($resolved->normalizedTags() as $tagName) {
$tag = Tag::firstOrCreate(['name' => $tagName]);
$tagIds[] = $tag->id;
}
$metadata->tags()->sync($tagIds);
} catch (TransferException $e) {
// 何らかの通信エラーによってメタデータの取得に失敗した時、とりあえずエラーログにURLを残す
Log::error(self::class . ': メタデータの取得に失敗 URL=' . $url);
throw $e;
} catch (\Exception $e) {
$metadata->storeException(now(), $e);
$metadata->save();
throw new UncaughtResolverException(implode(': ', [
$metadata->error_count . '回目のメタデータ取得失敗', get_class($e), $e->getMessage()
]), 0, $e);
}
$metadata->fill([
'title' => $resolved->title,
'description' => $resolved->description,
'image' => $resolved->image,
'expires_at' => $resolved->expires_at
]);
$metadata->clearError();
$metadata->save();
$tagIds = [];
foreach ($resolved->normalizedTags() as $tagName) {
$tag = Tag::firstOrCreate(['name' => $tagName]);
$tagIds[] = $tag->id;
}
$metadata->tags()->sync($tagIds);
}
DB::commit();
return $metadata;
});
} catch (UncaughtResolverException $e) {
// Metadataにエラー情報を記録するため
DB::commit();
throw $e;
} catch (\Exception $e) {
DB::rollBack();
throw $e;
}
}
}

36
database/migrations/2020_08_10_114944_add_error_data_to_metadata.php Normal file
View File

@ -0,0 +1,36 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class AddErrorDataToMetadata extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('metadata', function (Blueprint $table) {
$table->timestamp('error_at')->nullable();
$table->string('error_exception_class')->nullable();
$table->integer('error_http_code')->nullable();
$table->text('error_body')->nullable();
$table->integer('error_count')->default(0);
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('metadata', function (Blueprint $table) {
$table->dropColumn(['error_at', 'error_exception_class', 'error_http_code', 'error_body', 'error_count']);
});
}
}

182
tests/Unit/Services/MetadataResolverServiceTest.php Normal file
View File

@ -0,0 +1,182 @@
<?php
namespace Tests\Unit\Services;
use App\MetadataResolver\MetadataResolver;
use App\MetadataResolver\ResolverCircuitBreakException;
use App\MetadataResolver\UncaughtResolverException;
use App\Services\MetadataResolveService;
use Carbon\Carbon;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\ServerException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Mockery\MockInterface;
use Tests\TestCase;
class MetadataResolverServiceTest extends TestCase
{
use RefreshDatabase;
protected function setUp(): void
{
parent::setUp();
$this->seed();
Carbon::setTestNow('2020-07-21 19:19:19');
}
protected function tearDown(): void
{
parent::tearDown();
Carbon::setTestNow();
}
public function testOnRuntimeException()
{
$this->mock(MetadataResolver::class, function (MockInterface $mock) {
$mock->shouldReceive('resolve')->andReturnUsing(function ($url) {
throw new \RuntimeException('Something happened!');
});
});
try {
$service = app()->make(MetadataResolveService::class);
$service->execute('http://example.com');
} catch (UncaughtResolverException $e) {
$this->assertDatabaseHas('metadata', [
'url' => 'http://example.com',
'error_at' => new Carbon('2020-07-21 19:19:19'),
'error_count' => 1,
'error_exception_class' => \RuntimeException::class,
'error_http_code' => null,
'error_body' => 'Something happened!',
]);
return;
}
$this->fail();
}
public function testOnHttpClientError()
{
$handler = HandlerStack::create(new MockHandler([new Response(404)]));
$client = new Client(['handler' => $handler]);
$this->instance(Client::class, $client);
try {
$service = app()->make(MetadataResolveService::class);
$service->execute('http://example.com');
} catch (UncaughtResolverException $e) {
$this->assertDatabaseHas('metadata', [
'url' => 'http://example.com',
'error_at' => new Carbon('2020-07-21 19:19:19'),
'error_count' => 1,
'error_exception_class' => ClientException::class,
'error_http_code' => 404,
]);
return;
}
$this->fail();
}
public function testOnHttpServerError()
{
$handler = HandlerStack::create(new MockHandler([new Response(503), new Response(503)]));
$client = new Client(['handler' => $handler]);
$this->instance(Client::class, $client);
try {
$service = app()->make(MetadataResolveService::class);
$service->execute('http://example.com');
} catch (UncaughtResolverException $e) {
$this->assertDatabaseHas('metadata', [
'url' => 'http://example.com',
'error_at' => new Carbon('2020-07-21 19:19:19'),
'error_count' => 1,
'error_exception_class' => ServerException::class,
'error_http_code' => 503,
]);
return;
}
$this->fail();
}
public function testCircuitBreak()
{
$this->mock(MetadataResolver::class, function (MockInterface $mock) {
$mock->shouldReceive('resolve')->andReturnUsing(function ($url) {
throw new \RuntimeException('Something happened!');
});
});
try {
for ($i = 0; $i < 6; $i++) {
try {
$service = app()->make(MetadataResolveService::class);
$service->execute('http://example.com');
} catch (UncaughtResolverException $e) {
}
}
} catch (ResolverCircuitBreakException $e) {
$this->assertDatabaseHas('metadata', [
'url' => 'http://example.com',
'error_at' => new Carbon('2020-07-21 19:19:19'),
'error_count' => 5,
'error_exception_class' => \RuntimeException::class,
'error_http_code' => null,
'error_body' => 'Something happened!',
]);
return;
}
$this->fail();
}
public function testOnResurrect()
{
$successBody = <<<HTML
<!doctype html>
<html lang="ja">
<head>
<meta charset="UTF-8">
<meta name="og:title" content="OGP Title">
<meta name="og:description" content="OGP Description">
<title>Test Document</title>
</head>
<body>
</body>
</html>
HTML;
$handler = HandlerStack::create(new MockHandler([
new Response(404),
new Response(200, ['Content-Type' => 'text/html'], $successBody),
]));
$client = new Client(['handler' => $handler]);
$this->instance(Client::class, $client);
for ($i = 0; $i < 2; $i++) {
try {
$service = app()->make(MetadataResolveService::class);
$service->execute('http://example.com');
} catch (UncaughtResolverException $e) {
}
}
$this->assertDatabaseHas('metadata', [
'url' => 'http://example.com',
'title' => 'OGP Title',
'description' => 'OGP Description',
'image' => '',
'error_at' => null,
'error_count' => 0,
'error_exception_class' => null,
'error_http_code' => null,
'error_body' => null,
]);
}
}