v2.0.0 - AI-powered paper extraction without RSS

Major Features:
- AI-based paper extraction: Extract papers directly from journal web pages using Claude/OpenAI
- No RSS required: Works with any journal website that lists papers
- Automatic data normalization: AI handles date formats, DOI extraction, author parsing
- Background PDF processing: Queue-based PDF download and text extraction

Changes:
- AiRssGeneratorService: New extractPapersWithAi() for direct AI extraction
- RssFetcherService: Separate processing paths for RSS and AI-generated feeds
- PaperController: COALESCE sorting to handle NULL published_date
- PaperList: Default date filter changed to 'all'
- New migrations for jobs table and pdf_status column

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: dbym4820

.env.example

@@ -18,6 +18,10 @@ DB_SOCKET=
 # MAMP MySQL 5.7: /Applications/MAMP/Library/bin/mysql57/bin/mysql
 MYSQL_BIN=
 
+# キュー設定（PDF処理などの非同期ジョブ用）
+# database: DBを使用（推奨），sync: 同期処理（デバッグ用）
+QUEUE_CONNECTION=database
+
 # 初期管理者ユーザー（マイグレーション時に自動作成）
 ADMIN_USER_ID=
 ADMIN_USERNAME=

app/Http/Controllers/AdminController.php

@@ -149,11 +149,16 @@ public function createJournal(Request $request): JsonResponse
 
         // 初回フェッチを実行
         if ($journal->isAiGenerated()) {
-            // AI生成の場合
-            $fetchResult = $this->aiRssGenerator->generateFeed($journal, $user);
+            // AI生成の場合：ページ構造を解析してセレクタを保存
+            $setupResult = $this->aiRssGenerator->setupFeed($journal, $user);
             $message = '論文誌を追加しました';
-            if ($fetchResult['success']) {
-                $message .= '（' . ($fetchResult['papers_count'] ?? 0) . '件の論文を検出）';
+            if ($setupResult['success']) {
+                // HTMLからパースした論文情報をデータベースに登録
+                $fetchResult = $this->rssFetcher->fetchJournal($journal);
+                $newPapers = $fetchResult['new_papers'] ?? 0;
+                $message .= '（' . $newPapers . '件の論文を登録）';
+            } else {
+                $fetchResult = $setupResult;
             }
         } else {
             // 通常RSSの場合

app/Http/Controllers/GeneratedRssController.php

@@ -3,63 +3,24 @@
 namespace App\Http\Controllers;
 
 use Illuminate\Http\Request;
-use Illuminate\Http\Response;
 use Illuminate\Http\JsonResponse;
-use App\Models\GeneratedFeed;
 use App\Models\Journal;
 use App\Services\AiRssGeneratorService;
+use App\Services\RssFetcherService;
 
 class GeneratedRssController extends Controller
 {
-    private AiRssGeneratorService $rssGenerator;
+    private AiRssGeneratorService $aiGenerator;
+    private RssFetcherService $rssFetcher;
 
-    public function __construct(AiRssGeneratorService $rssGenerator)
+    public function __construct(AiRssGeneratorService $aiGenerator, RssFetcherService $rssFetcher)
     {
-        $this->rssGenerator = $rssGenerator;
+        $this->aiGenerator = $aiGenerator;
+        $this->rssFetcher = $rssFetcher;
     }
 
     /**
-     * Serve RSS feed by feed token (public, no authentication required)
-     * Dynamically fetches and parses the source page using saved selectors
-     */
-    public function serve(string $feedToken): Response
-    {
-        $feed = GeneratedFeed::where('feed_token', $feedToken)
-            ->with('journal')
-            ->first();
-
-        if (!$feed) {
-            return response('Feed not found', 404)
-                ->header('Content-Type', 'text/plain');
-        }
-
-        // Check if we have valid selectors
-        $selectors = $feed->extraction_config['selectors'] ?? null;
-        if (!$selectors || empty($selectors['title'])) {
-            return response('Feed not configured. Please regenerate the feed.', 503)
-                ->header('Content-Type', 'text/plain');
-        }
-
-        // Dynamically generate RSS by fetching and parsing the source page
-        try {
-            $rssXml = $this->rssGenerator->generateRssDynamically($feed);
-
-            return response($rssXml)
-                ->header('Content-Type', 'application/rss+xml; charset=utf-8')
-                ->header('Cache-Control', 'public, max-age=1800'); // Cache for 30 minutes
-        } catch (\Exception $e) {
-            \Log::error('Failed to generate RSS dynamically', [
-                'feed_token' => $feedToken,
-                'error' => $e->getMessage(),
-            ]);
-
-            return response('Failed to fetch feed: ' . $e->getMessage(), 503)
-                ->header('Content-Type', 'text/plain');
-        }
-    }
-
-    /**
-     * Regenerate feed (requires authentication)
+     * Reanalyze page structure and fetch papers (requires authentication)
      */
     public function regenerate(Request $request, string $journalId): JsonResponse
     {
@@ -81,21 +42,30 @@ public function regenerate(Request $request, string $journalId): JsonResponse
             ], 400);
         }
 
-        $result = $this->rssGenerator->generateFeed($journal, $user);
+        // AIでページ構造を再解析してセレクタを更新
+        $result = $this->aiGenerator->reanalyzeStructure($journal, $user);
 
         if (!$result['success']) {
-            return response()->json([
+            $response = [
                 'success' => false,
                 'error' => $result['error'],
-            ], 400);
+            ];
+            if (!empty($result['debug'])) {
+                $response['debug'] = $result['debug'];
+            }
+            return response()->json($response, 400);
         }
 
+        // HTMLからパースした論文情報をデータベースに登録
+        $fetchResult = $this->rssFetcher->fetchJournal($journal);
+        $newPapers = $fetchResult['new_papers'] ?? 0;
+
         return response()->json([
             'success' => true,
-            'message' => 'Feed regenerated successfully',
+            'message' => 'ページを再解析しました（' . $newPapers . '件の新規論文を登録）',
             'papers_count' => $result['papers_count'],
-            'feed_token' => $result['feed_token'],
-            'provider' => $result['provider'] ?? ($result['method'] === 'selector' ? 'selector' : null),
+            'new_papers' => $newPapers,
+            'provider' => $result['provider'] ?? null,
         ]);
     }
 
@@ -134,9 +104,9 @@ public function testPage(Request $request): JsonResponse
 
         // Use auto-redirect version if enabled
         if ($autoRedirect) {
-            $result = $this->rssGenerator->testPageAnalysisWithRedirect($url, $user);
+            $result = $this->aiGenerator->testPageAnalysisWithRedirect($url, $user);
         } else {
-            $result = $this->rssGenerator->testPageAnalysis($url, $user);
+            $result = $this->aiGenerator->testPageAnalysis($url, $user);
         }
 
         $response = [

app/Http/Controllers/JournalController.php

@@ -39,7 +39,6 @@ public function index(Request $request): JsonResponse
                 if ($j->generatedFeed) {
                     $data['generated_feed'] = [
                         'id' => $j->generatedFeed->id,
-                        'feed_token' => $j->generatedFeed->feed_token,
                         'source_url' => $j->generatedFeed->source_url,
                         'ai_provider' => $j->generatedFeed->ai_provider,
                         'ai_model' => $j->generatedFeed->ai_model,
@@ -94,7 +93,6 @@ public function show(Request $request, string $id): JsonResponse
         if ($journal->generatedFeed) {
             $response['generated_feed'] = [
                 'id' => $journal->generatedFeed->id,
-                'feed_token' => $journal->generatedFeed->feed_token,
                 'source_url' => $journal->generatedFeed->source_url,
                 'ai_provider' => $journal->generatedFeed->ai_provider,
                 'ai_model' => $journal->generatedFeed->ai_model,

app/Http/Controllers/PaperController.php

@@ -7,6 +7,7 @@
 use Illuminate\Support\Facades\Storage;
 use Symfony\Component\HttpFoundation\StreamedResponse;
 use App\Models\Paper;
+use App\Services\QueueRunnerService;
 use Illuminate\Support\Facades\DB;
 
 class PaperController extends Controller
@@ -42,7 +43,8 @@ public function index(Request $request): JsonResponse
         $offset = (int) ($request->offset ?? 0);
 
         $total = $query->count();
-        $papers = $query->orderBy('published_date', 'desc')
+        // Use COALESCE to handle NULL published_date (AI-generated papers may not have dates)
+        $papers = $query->orderByRaw('COALESCE(published_date, DATE(fetched_at)) DESC')
             ->orderBy('fetched_at', 'desc')
             ->offset($offset)
             ->limit($limit)
@@ -154,6 +156,51 @@ public function downloadPdf(Request $request, int $id): StreamedResponse|JsonRes
         ]);
     }
 
+    /**
+     * PDF処理状況を確認し、必要に応じてワーカーを再起動
+     * 各論文のpdf_statusも返す
+     */
+    public function processingStatus(Request $request): JsonResponse
+    {
+        $user = $request->attributes->get('user');
+
+        // ユーザーの論文でPDF処理に関係するもの（pending/processing/completed/failed）のIDとステータスを取得
+        $paperStatuses = Paper::forUser($user->id)
+            ->whereNotNull('pdf_status')
+            ->select('id', 'pdf_status', 'pdf_path')
+            ->get()
+            ->map(function ($paper) {
+                return [
+                    'id' => $paper->id,
+                    'pdf_status' => $paper->pdf_status,
+                    'has_local_pdf' => !empty($paper->pdf_path),
+                ];
+            });
+
+        $processingCount = $paperStatuses->whereIn('pdf_status', ['pending', 'processing'])->count();
+
+        // キュー内のジョブ数
+        $pendingJobs = QueueRunnerService::getPendingJobCount('pdf-processing');
+
+        // ワーカーの状態
+        $workerRunning = QueueRunnerService::isWorkerRunning('pdf-processing');
+        $workerStarted = false;
+
+        // ジョブがある場合は常にワーカー起動を試みる
+        if ($pendingJobs > 0) {
+            $workerStarted = QueueRunnerService::startWorkerIfNeeded('pdf-processing');
+        }
+
+        return response()->json([
+            'success' => true,
+            'processing_count' => $processingCount,
+            'pending_jobs' => $pendingJobs,
+            'worker_running' => $workerRunning || $workerStarted,
+            'worker_started' => $workerStarted,
+            'paper_statuses' => $paperStatuses,
+        ]);
+    }
+
     private function formatPaper(Paper $paper, bool $detailed = false): array
     {
         $data = [
@@ -182,6 +229,7 @@ private function formatPaper(Paper $paper, bool $detailed = false): array
             'full_text_source' => $paper->full_text_source,
             'pdf_url' => $paper->pdf_url,
             'has_local_pdf' => $paper->hasLocalPdf(),
+            'pdf_status' => $paper->pdf_status,
             // Always include summaries for frontend to show existing summaries
             'summaries' => $paper->summaries->map(function ($s) {
                 return [

app/Jobs/ProcessPaperFullTextJob.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Paper;
+use App\Services\FullTextFetcherService;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ProcessPaperFullTextJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * ジョブのタイムアウト（秒）
+     * PDF解析に十分な時間を確保
+     */
+    public int $timeout = 300; // 5分
+
+    /**
+     * 失敗前の試行回数
+     */
+    public int $tries = 2;
+
+    /**
+     * 再試行までの待機秒数
+     */
+    public int $backoff = 60;
+
+    protected int $paperId;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(int $paperId)
+    {
+        $this->paperId = $paperId;
+        $this->onQueue('pdf-processing');
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(FullTextFetcherService $fullTextFetcher): void
+    {
+        $paper = Paper::find($this->paperId);
+
+        if (!$paper) {
+            Log::warning("ProcessPaperFullTextJob: Paper {$this->paperId} not found");
+            return;
+        }
+
+        // 既に処理済みの場合はスキップ
+        if ($paper->pdf_status === 'completed') {
+            Log::debug("ProcessPaperFullTextJob: Paper {$this->paperId} already completed");
+            return;
+        }
+
+        // ステータスを「処理中」に更新
+        $paper->update(['pdf_status' => 'processing']);
+
+        Log::info("ProcessPaperFullTextJob: Processing paper {$this->paperId} - {$paper->title}");
+
+        try {
+            $result = $fullTextFetcher->fetchFullText($paper);
+
+            if ($result['success']) {
+                $updateData = [
+                    'full_text' => $result['text'],
+                    'full_text_source' => $result['source'],
+                    'full_text_fetched_at' => now(),
+                    'pdf_status' => 'completed',
+                ];
+
+                if (!empty($result['pdf_url'])) {
+                    $updateData['pdf_url'] = $result['pdf_url'];
+                }
+                if (!empty($result['pdf_path'])) {
+                    $updateData['pdf_path'] = $result['pdf_path'];
+                }
+
+                $paper->update($updateData);
+
+                Log::info("ProcessPaperFullTextJob: Successfully processed paper {$this->paperId}");
+            } else {
+                // PDF取得に失敗した場合もステータスを更新（再試行対象外）
+                $paper->update(['pdf_status' => 'failed']);
+                Log::debug("ProcessPaperFullTextJob: Failed to fetch full text for paper {$this->paperId}: " . ($result['error'] ?? 'Unknown error'));
+            }
+        } catch (\Throwable $e) {
+            Log::error("ProcessPaperFullTextJob: Exception processing paper {$this->paperId}: " . $e->getMessage());
+            throw $e; // 再試行のために例外を再スロー
+        }
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(\Throwable $exception): void
+    {
+        Log::error("ProcessPaperFullTextJob: Job failed for paper {$this->paperId}: " . $exception->getMessage());
+
+        // 最終的に失敗した場合はステータスを更新
+        $paper = Paper::find($this->paperId);
+        if ($paper) {
+            $paper->update(['pdf_status' => 'failed']);
+        }
+    }
+}