feat: add save-and-scan onboarding step and /api/opportunities/refresh (v1.1 steps 3&4)

- lib/refresh-opportunities.ts: shared fetch/score/save logic for single user
- api/onboarding: replace generate-keywords+save with save-and-scan (fire-and-forget refresh)
- api/opportunities/refresh: new POST endpoint for on-demand per-user scan

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>

Author: dongzhang84

app/api/onboarding/route.ts

@@ -2,6 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { createServerSupabaseClient } from '@/lib/supabase/server'
 import { generateKeywordsAndSubreddits } from '@/lib/keyword-generator'
 import { prisma } from '@/lib/db/client'
+import { refreshOpportunitiesForUser } from '@/lib/refresh-opportunities'
 
 export async function POST(request: NextRequest) {
   const supabase = await createServerSupabaseClient()
@@ -20,29 +21,17 @@ export async function POST(request: NextRequest) {
 
   const { step } = body
 
-  if (step === 'generate-keywords') {
-    const { productDescription, targetCustomer } = body as {
-      productDescription: string
-      targetCustomer: string
-    }
-
-    const result = await generateKeywordsAndSubreddits(productDescription)
-    return NextResponse.json(result)
-  }
+  if (step === 'save-and-scan') {
+    const { productDescription } = body as { productDescription: string }
 
-  if (step === 'save') {
-    const { productDescription, targetCustomer, keywords, subreddits } = body as {
-      productDescription: string
-      targetCustomer: string
-      keywords: string[]
-      subreddits: string[]
-    }
+    // Generate keywords and subreddits from product description (user never sees this)
+    const { keywords, subreddits } = await generateKeywordsAndSubreddits(productDescription)
 
+    // Save profile
     await prisma.profile.upsert({
       where: { id: user.id },
       update: {
         productDescription,
-        targetCustomer,
         keywords,
         subreddits,
         onboardingComplete: true,
@@ -52,7 +41,6 @@ export async function POST(request: NextRequest) {
         email: user.email ?? '',
         password: '',
         productDescription,
-        targetCustomer,
         keywords,
         subreddits,
         onboardingComplete: true,
@@ -61,8 +49,20 @@ export async function POST(request: NextRequest) {
       },
     })
 
+    // Fire and forget — scan runs in background, we return immediately
+    refreshOpportunitiesForUser(user.id).catch((err) =>
+      console.error('[onboarding] background refresh failed:', err)
+    )
+
     return NextResponse.json({ success: true })
   }
 
+  // Legacy steps kept for reference but no longer used by the UI
+  if (step === 'generate-keywords') {
+    const { productDescription } = body as { productDescription: string }
+    const result = await generateKeywordsAndSubreddits(productDescription)
+    return NextResponse.json(result)
+  }
+
   return NextResponse.json({ error: 'Unknown step' }, { status: 400 })
 }

app/api/opportunities/refresh/route.ts

@@ -0,0 +1,23 @@
+import { NextResponse } from 'next/server'
+import { createServerSupabaseClient } from '@/lib/supabase/server'
+import { refreshOpportunitiesForUser } from '@/lib/refresh-opportunities'
+
+export const maxDuration = 60
+
+export async function POST() {
+  const supabase = await createServerSupabaseClient()
+  const { data: { user } } = await supabase.auth.getUser()
+
+  if (!user) {
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+  }
+
+  try {
+    const opportunitiesSaved = await refreshOpportunitiesForUser(user.id)
+    return NextResponse.json({ success: true, opportunitiesSaved })
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Unknown error'
+    console.error('[refresh] Unhandled error:', message)
+    return NextResponse.json({ success: false, error: message })
+  }
+}

lib/refresh-opportunities.ts

@@ -0,0 +1,135 @@
+import { prisma } from '@/lib/db/client'
+import { fetchSubredditPosts, type RedditPost } from '@/lib/reddit'
+import { fetchHNStories, type HNStory } from '@/lib/hn'
+import { scorePosts, type PostToScore } from '@/lib/scorer'
+
+interface NormalizedPost {
+  externalId: string
+  title: string
+  body: string
+  author: string
+  score: number
+  commentCount: number
+  url: string
+  postedAt: Date
+  platform: string
+  subreddit: string | null
+}
+
+function fromReddit(post: RedditPost): NormalizedPost {
+  return { ...post, platform: 'reddit', subreddit: post.subreddit }
+}
+
+function fromHN(story: HNStory): NormalizedPost {
+  return { ...story, platform: 'hn', subreddit: null }
+}
+
+function matchesKeywords(post: NormalizedPost, keywords: string[]): boolean {
+  if (keywords.length === 0) return false
+  const haystack = `${post.title} ${post.body}`.toLowerCase()
+  return keywords.some((kw) => {
+    const phrase = kw.toLowerCase()
+    if (haystack.includes(phrase)) return true
+    const words = phrase.split(/\s+/).filter((w) => w.length > 4)
+    return words.some((word) => haystack.includes(word))
+  })
+}
+
+export async function refreshOpportunitiesForUser(userId: string): Promise<number> {
+  const profile = await prisma.profile.findUnique({
+    where: { id: userId },
+    select: { keywords: true, subreddits: true, productDescription: true, targetCustomer: true },
+  })
+
+  if (!profile || profile.keywords.length === 0) {
+    console.log(`[refresh] User ${userId} — no keywords, skipping`)
+    return 0
+  }
+
+  // Fetch Reddit posts for each of the user's subreddits
+  const redditPosts: NormalizedPost[] = []
+  await Promise.all(
+    profile.subreddits.map(async (subreddit) => {
+      try {
+        const posts = await fetchSubredditPosts(subreddit)
+        redditPosts.push(...posts.map(fromReddit))
+        console.log(`[refresh] r/${subreddit}: fetched ${posts.length} post(s)`)
+      } catch {
+        console.error(`[refresh] Failed to fetch r/${subreddit}`)
+      }
+    })
+  )
+
+  // Fetch HN stories
+  let hnStories: NormalizedPost[] = []
+  try {
+    hnStories = (await fetchHNStories(150)).map(fromHN)
+    console.log(`[refresh] HN: fetched ${hnStories.length} story(ies)`)
+  } catch {
+    console.error('[refresh] Failed to fetch HN stories')
+  }
+
+  const candidates = [...redditPosts, ...hnStories]
+  const matched = candidates.filter((post) => matchesKeywords(post, profile.keywords))
+
+  console.log(`[refresh] User ${userId} — candidates: ${candidates.length}, matched: ${matched.length}`)
+
+  if (matched.length === 0) return 0
+
+  // Cap at 30 most recent
+  const capped = matched
+    .sort((a, b) => b.postedAt.getTime() - a.postedAt.getTime())
+    .slice(0, 30)
+
+  const postsToScore: PostToScore[] = capped.map((post) => ({
+    externalId: post.externalId,
+    title: post.title,
+    body: post.body,
+    subreddit: post.subreddit,
+    postedAt: post.postedAt,
+    commentCount: post.commentCount,
+  }))
+
+  let scored = await scorePosts(
+    postsToScore,
+    profile.productDescription ?? '',
+    profile.targetCustomer ?? ''
+  ).catch((err) => {
+    console.error(`[refresh] Scoring failed for user ${userId}:`, err)
+    return []
+  })
+
+  scored = scored.filter((p) => p.relevanceScore >= 40)
+  console.log(`[refresh] User ${userId} — scored (relevance>=40): ${scored.length}`)
+
+  if (scored.length === 0) return 0
+
+  const metaMap = new Map(capped.map((p) => [p.externalId, p]))
+
+  const result = await prisma.opportunity.createMany({
+    data: scored.map((p) => {
+      const meta = metaMap.get(p.externalId)!
+      return {
+        userId,
+        platform: meta.platform,
+        externalId: p.externalId,
+        url: meta.url,
+        title: p.title,
+        body: p.body || null,
+        subreddit: p.subreddit,
+        author: meta.author,
+        score: meta.score,
+        commentCount: p.commentCount,
+        postedAt: p.postedAt,
+        relevanceScore: p.relevanceScore,
+        intentLevel: p.intentLevel,
+        reasoning: p.reasoning,
+        suggestedReplies: [],
+      }
+    }),
+    skipDuplicates: true,
+  })
+
+  console.log(`[refresh] User ${userId} — saved ${result.count} new opportunity(ies)`)
+  return result.count
+}