feat: onboarding config, reset onboarding, xai agentic migration (#701)01-24-feat_onboarding_config_reset_onboarding_xai_agentic_migration

- Created a new `useOrgOnboarding` hook that uses `org.metadata.isOnboarded` to track onboarding state
- Updated the home page to conditionally use either the old localStorage-based onboarding or the new DB-backed onboarding based on feature flag
- Added a "Restart Onboarding" option in the user dropdown menu
- Improved the onboarding chat sidebar with per-link loading indicators
- Enhanced the X/Twitter research API to better handle different URL formats
- Updated the integrations step to use the new onboarding completion method
- Added `updateOrgMetadata` function to the auth context for easier metadata updates

5 files changed, 179 insertions, 84 deletions

diff --git a/apps/web/app/(navigation)/page.tsx b/apps/web/app/(navigation)/page.tsx
index 73da4f3c..39b6f47c 100644
--- a/apps/web/app/(navigation)/page.tsx
+++ b/apps/web/app/(navigation)/page.tsx
@@ -1,10 +1,11 @@
 "use client"
 
 import { useOnboardingStorage } from "@hooks/use-onboarding-storage"
+import { useOrgOnboarding } from "@hooks/use-org-onboarding"
 import { useAuth } from "@lib/auth-context"
 import { ChevronsDown, LoaderIcon } from "lucide-react"
 import { useRouter } from "next/navigation"
-import { useEffect } from "react"
+import { useEffect, useMemo } from "react"
 import { InstallPrompt } from "@/components/install-prompt"
 import { ChromeExtensionButton } from "@/components/chrome-extension-button"
 import { ChatInput } from "@/components/chat-input"
@@ -14,11 +15,37 @@ import { useFeatureFlagEnabled } from "posthog-js/react"
 
 export default function Page() {
 	const { user, session } = useAuth()
-	const { shouldShowOnboarding, isLoading: onboardingLoading } =
-		useOnboardingStorage()
 	const router = useRouter()
 	const flagEnabled = useFeatureFlagEnabled("nova-alpha-access")
 
+	// TODO: remove this flow after the feature flag is removed
+	// Old app: localStorage-backed onboarding
+	const {
+		shouldShowOnboarding: shouldShowOldOnboarding,
+		isLoading: oldOnboardingLoading,
+	} = useOnboardingStorage()
+
+	// New app: DB-backed onboarding (org.metadata.isOnboarded)
+	const {
+		shouldShowOnboarding: shouldShowNewOnboarding,
+		isLoading: newOnboardingLoading,
+	} = useOrgOnboarding()
+
+	// Select the appropriate onboarding state based on feature flag
+	const isOnboardingLoading = useMemo(() => {
+		if (flagEnabled) {
+			return newOnboardingLoading
+		}
+		return oldOnboardingLoading
+	}, [flagEnabled, newOnboardingLoading, oldOnboardingLoading])
+
+	const shouldShowOnboarding = useMemo(() => {
+		if (flagEnabled) {
+			return shouldShowNewOnboarding()
+		}
+		return shouldShowOldOnboarding()
+	}, [flagEnabled, shouldShowNewOnboarding, shouldShowOldOnboarding])
+
 	useEffect(() => {
 		const url = new URL(window.location.href)
 		const authenticateChromeExtension = url.searchParams.get(
@@ -46,16 +73,16 @@ export default function Page() {
 	}, [user, session])
 
 	useEffect(() => {
-		if (user && !onboardingLoading && shouldShowOnboarding()) {
+		if (user && !isOnboardingLoading && shouldShowOnboarding) {
 			if (flagEnabled) {
 				router.push("/new/onboarding?step=input&flow=welcome")
 			} else {
 				router.push("/onboarding")
 			}
 		}
-	}, [user, shouldShowOnboarding, onboardingLoading, router, flagEnabled])
+	}, [user, shouldShowOnboarding, isOnboardingLoading, router, flagEnabled])
 
-	if (!user || onboardingLoading) {
+	if (!user || isOnboardingLoading) {
 		return (
 			<div className="min-h-screen flex items-center justify-center bg-[#0f1419]">
 				<div className="flex flex-col items-center gap-4">
@@ -66,7 +93,7 @@ export default function Page() {
 		)
 	}
 
-	if (shouldShowOnboarding()) {
+	if (shouldShowOnboarding) {
 		return null
 	}
 
diff --git a/apps/web/app/api/onboarding/research/route.ts b/apps/web/app/api/onboarding/research/route.ts
index 5e9b933e..67bf4654 100644
--- a/apps/web/app/api/onboarding/research/route.ts
+++ b/apps/web/app/api/onboarding/research/route.ts
@@ -7,11 +7,22 @@ interface ResearchRequest {
 	email?: string
 }
 
-// prompt to get user context from X/Twitter profile
-function finalPrompt(xUrl: string, userContext: string) {
+function extractHandle(url: string): string {
+	const cleaned = url
+		.toLowerCase()
+		.replace("https://x.com/", "")
+		.replace("https://twitter.com/", "")
+		.replace("http://x.com/", "")
+		.replace("http://twitter.com/", "")
+		.replace("@", "")
+
+	return (cleaned.split("/")[0] ?? cleaned).split("?")[0] ?? cleaned
+}
+
+function finalPrompt(handle: string, userContext: string) {
 	return `You are researching a user based on their X/Twitter profile to help personalize their experience.
 
-X/Twitter Profile URL: ${xUrl}${userContext}
+X Handle: @${handle}${userContext}
 
 Please analyze this X/Twitter profile and provide a comprehensive but concise summary of the user. Include:
 - Professional background and current role (if available)
@@ -29,18 +40,12 @@ export async function POST(req: Request) {
 
 		if (!xUrl?.trim()) {
 			return Response.json(
-				{ error: "X/Twitter URL is required" },
+				{ error: "X/Twitter URL or handle is required" },
 				{ status: 400 },
 			)
 		}
 
-		const lowerUrl = xUrl.toLowerCase()
-		if (!lowerUrl.includes("x.com") && !lowerUrl.includes("twitter.com")) {
-			return Response.json(
-				{ error: "URL must be an X/Twitter profile link" },
-				{ status: 400 },
-			)
-		}
+		const handle = extractHandle(xUrl)
 
 		const contextParts: string[] = []
 		if (name) contextParts.push(`Name: ${name}`)
@@ -51,29 +56,13 @@ export async function POST(req: Request) {
 				: ""
 
 		const { text } = await generateText({
-			model: xai("grok-4-1-fast-reasoning"),
-			prompt: finalPrompt(xUrl, userContext),
-			providerOptions: {
-				xai: {
-					searchParameters: {
-						mode: "on",
-						sources: [
-							{
-								type: "web",
-								safeSearch: true,
-							},
-							{
-								type: "x",
-								includedXHandles: [
-									lowerUrl
-										.replace("https://x.com/", "")
-										.replace("https://twitter.com/", ""),
-								],
-								postFavoriteCount: 10,
-							},
-						],
-					},
-				},
+			model: xai.responses("grok-4-fast"),
+			prompt: finalPrompt(handle, userContext),
+			tools: {
+				web_search: xai.tools.webSearch(),
+				x_search: xai.tools.xSearch({
+					allowedXHandles: [handle],
+				}),
 			},
 		})
 
diff --git a/apps/web/components/new/header.tsx b/apps/web/components/new/header.tsx
index 9691f733..4275e5f8 100644
--- a/apps/web/components/new/header.tsx
+++ b/apps/web/components/new/header.tsx
@@ -15,6 +15,7 @@ import {
 	HelpCircle,
 	MenuIcon,
 	MessageCircleIcon,
+	RotateCcw,
 } from "lucide-react"
 import { Button } from "@ui/components/button"
 import { cn } from "@lib/utils"
@@ -34,6 +35,7 @@ import { useRouter } from "next/navigation"
 import Link from "next/link"
 import { SpaceSelector } from "./space-selector"
 import { useIsMobile } from "@hooks/use-mobile"
+import { useOrgOnboarding } from "@hooks/use-org-onboarding"
 
 interface HeaderProps {
 	onAddMemory?: () => void
@@ -53,6 +55,12 @@ export function Header({
 	const { switchProject } = useProjectMutations()
 	const router = useRouter()
 	const isMobile = useIsMobile()
+	const { resetOrgOnboarded } = useOrgOnboarding()
+
+	const handleTryOnboarding = () => {
+		resetOrgOnboarded()
+		router.push("/new/onboarding?step=input&flow=welcome")
+	}
 
 	const displayName =
 		user?.displayUsername ||
@@ -316,6 +324,13 @@ export function Header({
 								<Settings className="h-4 w-4 text-[#737373]" />
 								Settings
 							</DropdownMenuItem>
+							<DropdownMenuItem
+								onClick={handleTryOnboarding}
+								className="px-3 py-2.5 rounded-md hover:bg-[#293952]/40 cursor-pointer text-white text-sm font-medium gap-2"
+							>
+								<RotateCcw className="h-4 w-4 text-[#737373]" />
+								Restart Onboarding
+							</DropdownMenuItem>
 							<DropdownMenuSeparator className="bg-[#2E3033]" />
 							<DropdownMenuItem
 								asChild
diff --git a/apps/web/components/new/onboarding/setup/chat-sidebar.tsx b/apps/web/components/new/onboarding/setup/chat-sidebar.tsx
index 22e8cae1..47af432d 100644
--- a/apps/web/components/new/onboarding/setup/chat-sidebar.tsx
+++ b/apps/web/components/new/onboarding/setup/chat-sidebar.tsx
@@ -6,7 +6,13 @@ import { useChat } from "@ai-sdk/react"
 import { DefaultChatTransport } from "ai"
 import NovaOrb from "@/components/nova/nova-orb"
 import { Button } from "@ui/components/button"
-import { PanelRightCloseIcon, SendIcon, CheckIcon, XIcon } from "lucide-react"
+import {
+	PanelRightCloseIcon,
+	SendIcon,
+	CheckIcon,
+	XIcon,
+	Loader2,
+} from "lucide-react"
 import { collectValidUrls } from "@/lib/url-helpers"
 import { $fetch } from "@lib/api"
 import { cn } from "@lib/utils"
@@ -61,10 +67,14 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 		"correct" | "incorrect" | null
 	>(null)
 	const [isConfirmed, setIsConfirmed] = useState(false)
+	const [processingByUrl, setProcessingByUrl] = useState<Record<string, boolean>>(
+		{},
+	)
 	const displayedMemoriesRef = useRef<Set<string>>(new Set())
 	const contextInjectedRef = useRef(false)
 	const draftsBuiltRef = useRef(false)
 	const isProcessingRef = useRef(false)
+	const draftRequestIdRef = useRef(0)
 
 	const {
 		messages: chatMessages,
@@ -225,9 +235,27 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 
 		if (!hasContent) return
 
+		const requestId = ++draftRequestIdRef.current
+
 		setIsFetchingDrafts(true)
 		const drafts: DraftDoc[] = []
 
+		const urls = collectValidUrls(formData.linkedin, formData.otherLinks)
+		const allProcessingUrls: string[] = [...urls]
+		if (formData.twitter) {
+			allProcessingUrls.push(formData.twitter)
+		}
+
+		if (allProcessingUrls.length > 0) {
+			setProcessingByUrl((prev) => {
+				const next = { ...prev }
+				for (const url of allProcessingUrls) {
+					next[url] = true
+				}
+				return next
+			})
+		}
+
 		try {
 			if (formData.description?.trim()) {
 				drafts.push({
@@ -241,37 +269,66 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 				})
 			}
 
-			const urls = collectValidUrls(formData.linkedin, formData.otherLinks)
+			// Fetch each URL separately for per-link loading state
+			const linkPromises = urls.map(async (url) => {
+				try {
+					const response = await fetch("/api/onboarding/extract-content", {
+						method: "POST",
+						headers: { "Content-Type": "application/json" },
+						body: JSON.stringify({ urls: [url] }),
+					})
+					const data = await response.json()
+					return data.results?.[0] || null
+				} catch {
+					return null
+				} finally {
+					// Clear this URL's processing state
+					if (draftRequestIdRef.current === requestId) {
+						setProcessingByUrl((prev) => ({ ...prev, [url]: false }))
+					}
+				}
+			})
+
+			// Fetch X/Twitter research
+			const xResearchPromise = formData.twitter
+				? (async () => {
+						try {
+							const response = await fetch("/api/onboarding/research", {
+								method: "POST",
+								headers: { "Content-Type": "application/json" },
+								body: JSON.stringify({
+									xUrl: formData.twitter,
+									name: user?.name,
+									email: user?.email,
+								}),
+							})
+							if (!response.ok) return null
+							const data = await response.json()
+							return data?.text?.trim() || null
+						} catch {
+							return null
+						} finally {
+							// Clear twitter URL's processing state
+							if (draftRequestIdRef.current === requestId) {
+								setProcessingByUrl((prev) => ({
+									...prev,
+									[formData.twitter]: false,
+								}))
+							}
+						}
+					})()
+				: Promise.resolve(null)
 
 			const [exaResults, xResearchResult] = await Promise.all([
-				urls.length > 0
-					? fetch("/api/onboarding/extract-content", {
-							method: "POST",
-							headers: { "Content-Type": "application/json" },
-							body: JSON.stringify({ urls }),
-						})
-							.then((r) => r.json())
-							.then((data) => data.results || [])
-							.catch(() => [])
-					: Promise.resolve([]),
-				formData.twitter
-					? fetch("/api/onboarding/research", {
-							method: "POST",
-							headers: { "Content-Type": "application/json" },
-							body: JSON.stringify({
-								xUrl: formData.twitter,
-								name: user?.name,
-								email: user?.email,
-							}),
-						})
-							.then((r) => (r.ok ? r.json() : null))
-							.then((data) => data?.text?.trim() || null)
-							.catch(() => null)
-					: Promise.resolve(null),
+				Promise.all(linkPromises),
+				xResearchPromise,
 			])
 
+			// Guard against stale request completing after a newer one
+			if (draftRequestIdRef.current !== requestId) return
+
 			for (const result of exaResults) {
-				if (result.text || result.description) {
+				if (result && (result.text || result.description)) {
 					drafts.push({
 						kind: "link",
 						content: result.text || result.description || "",
@@ -304,7 +361,9 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 		} catch (error) {
 			console.warn("Error building draft docs:", error)
 		} finally {
-			setIsFetchingDrafts(false)
+			if (draftRequestIdRef.current === requestId) {
+				setIsFetchingDrafts(false)
+			}
 		}
 	}, [formData, user])
 
@@ -502,18 +561,23 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 										{msg.type === "formData" && (
 											<div className="bg-[#293952]/40 rounded-lg p-2 px-3 space-y-1 flex-1">
 												{msg.title && (
-													<h3
-														className="text-sm font-medium"
-														style={{
-															background:
-																"linear-gradient(90deg, #369BFD 0%, #36FDFD 30%, #36FDB5 100%)",
-															WebkitBackgroundClip: "text",
-															WebkitTextFillColor: "transparent",
-															backgroundClip: "text",
-														}}
-													>
-														{msg.title}
-													</h3>
+													<div className="flex items-center gap-2">
+														<h3
+															className="text-sm font-medium"
+															style={{
+																background:
+																	"linear-gradient(90deg, #369BFD 0%, #36FDFD 30%, #36FDB5 100%)",
+																WebkitBackgroundClip: "text",
+																WebkitTextFillColor: "transparent",
+																backgroundClip: "text",
+															}}
+														>
+															{msg.title}
+														</h3>
+														{msg.url && processingByUrl[msg.url] && (
+															<Loader2 className="h-3 w-3 animate-spin text-blue-400" />
+														)}
+													</div>
 												)}
 												{msg.url && (
 													<a
diff --git a/apps/web/components/new/onboarding/setup/integrations-step.tsx b/apps/web/components/new/onboarding/setup/integrations-step.tsx
index 03102b14..951f26c0 100644
--- a/apps/web/components/new/onboarding/setup/integrations-step.tsx
+++ b/apps/web/components/new/onboarding/setup/integrations-step.tsx
@@ -7,7 +7,7 @@ import { XBookmarksDetailView } from "@/components/new/onboarding/x-bookmarks-de
 import { useRouter } from "next/navigation"
 import { cn } from "@lib/utils"
 import { dmSansClassName } from "@/lib/fonts"
-import { useOnboardingStorage } from "@hooks/use-onboarding-storage"
+import { useOrgOnboarding } from "@hooks/use-org-onboarding"
 import { analytics } from "@/lib/analytics"
 
 const integrationCards = [
@@ -61,11 +61,11 @@ const integrationCards = [
 export function IntegrationsStep() {
 	const router = useRouter()
 	const [selectedCard, setSelectedCard] = useState<string | null>(null)
-	const { markOnboardingCompleted } = useOnboardingStorage()
+	const { markOrgOnboarded } = useOrgOnboarding()
 
 	const handleContinue = () => {
+		markOrgOnboarded()
 		analytics.onboardingCompleted()
-		markOnboardingCompleted()
 		router.push("/new")
 	}