feat: onboarding config, reset onboarding, xai agentic migration (#701)01-24-feat_onboarding_config_reset_onboarding_xai_agentic_migration

- Created a new `useOrgOnboarding` hook that uses `org.metadata.isOnboarded` to track onboarding state
- Updated the home page to conditionally use either the old localStorage-based onboarding or the new DB-backed onboarding based on feature flag
- Added a "Restart Onboarding" option in the user dropdown menu
- Improved the onboarding chat sidebar with per-link loading indicators
- Enhanced the X/Twitter research API to better handle different URL formats
- Updated the integrations step to use the new onboarding completion method
- Added `updateOrgMetadata` function to the auth context for easier metadata updates

3 files changed, 122 insertions, 43 deletions

diff --git a/apps/web/components/new/header.tsx b/apps/web/components/new/header.tsx
index 9691f733..4275e5f8 100644
--- a/apps/web/components/new/header.tsx
+++ b/apps/web/components/new/header.tsx
@@ -15,6 +15,7 @@ import {
 	HelpCircle,
 	MenuIcon,
 	MessageCircleIcon,
+	RotateCcw,
 } from "lucide-react"
 import { Button } from "@ui/components/button"
 import { cn } from "@lib/utils"
@@ -34,6 +35,7 @@ import { useRouter } from "next/navigation"
 import Link from "next/link"
 import { SpaceSelector } from "./space-selector"
 import { useIsMobile } from "@hooks/use-mobile"
+import { useOrgOnboarding } from "@hooks/use-org-onboarding"
 
 interface HeaderProps {
 	onAddMemory?: () => void
@@ -53,6 +55,12 @@ export function Header({
 	const { switchProject } = useProjectMutations()
 	const router = useRouter()
 	const isMobile = useIsMobile()
+	const { resetOrgOnboarded } = useOrgOnboarding()
+
+	const handleTryOnboarding = () => {
+		resetOrgOnboarded()
+		router.push("/new/onboarding?step=input&flow=welcome")
+	}
 
 	const displayName =
 		user?.displayUsername ||
@@ -316,6 +324,13 @@ export function Header({
 								<Settings className="h-4 w-4 text-[#737373]" />
 								Settings
 							</DropdownMenuItem>
+							<DropdownMenuItem
+								onClick={handleTryOnboarding}
+								className="px-3 py-2.5 rounded-md hover:bg-[#293952]/40 cursor-pointer text-white text-sm font-medium gap-2"
+							>
+								<RotateCcw className="h-4 w-4 text-[#737373]" />
+								Restart Onboarding
+							</DropdownMenuItem>
 							<DropdownMenuSeparator className="bg-[#2E3033]" />
 							<DropdownMenuItem
 								asChild
diff --git a/apps/web/components/new/onboarding/setup/chat-sidebar.tsx b/apps/web/components/new/onboarding/setup/chat-sidebar.tsx
index 22e8cae1..47af432d 100644
--- a/apps/web/components/new/onboarding/setup/chat-sidebar.tsx
+++ b/apps/web/components/new/onboarding/setup/chat-sidebar.tsx
@@ -6,7 +6,13 @@ import { useChat } from "@ai-sdk/react"
 import { DefaultChatTransport } from "ai"
 import NovaOrb from "@/components/nova/nova-orb"
 import { Button } from "@ui/components/button"
-import { PanelRightCloseIcon, SendIcon, CheckIcon, XIcon } from "lucide-react"
+import {
+	PanelRightCloseIcon,
+	SendIcon,
+	CheckIcon,
+	XIcon,
+	Loader2,
+} from "lucide-react"
 import { collectValidUrls } from "@/lib/url-helpers"
 import { $fetch } from "@lib/api"
 import { cn } from "@lib/utils"
@@ -61,10 +67,14 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 		"correct" | "incorrect" | null
 	>(null)
 	const [isConfirmed, setIsConfirmed] = useState(false)
+	const [processingByUrl, setProcessingByUrl] = useState<Record<string, boolean>>(
+		{},
+	)
 	const displayedMemoriesRef = useRef<Set<string>>(new Set())
 	const contextInjectedRef = useRef(false)
 	const draftsBuiltRef = useRef(false)
 	const isProcessingRef = useRef(false)
+	const draftRequestIdRef = useRef(0)
 
 	const {
 		messages: chatMessages,
@@ -225,9 +235,27 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 
 		if (!hasContent) return
 
+		const requestId = ++draftRequestIdRef.current
+
 		setIsFetchingDrafts(true)
 		const drafts: DraftDoc[] = []
 
+		const urls = collectValidUrls(formData.linkedin, formData.otherLinks)
+		const allProcessingUrls: string[] = [...urls]
+		if (formData.twitter) {
+			allProcessingUrls.push(formData.twitter)
+		}
+
+		if (allProcessingUrls.length > 0) {
+			setProcessingByUrl((prev) => {
+				const next = { ...prev }
+				for (const url of allProcessingUrls) {
+					next[url] = true
+				}
+				return next
+			})
+		}
+
 		try {
 			if (formData.description?.trim()) {
 				drafts.push({
@@ -241,37 +269,66 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 				})
 			}
 
-			const urls = collectValidUrls(formData.linkedin, formData.otherLinks)
+			// Fetch each URL separately for per-link loading state
+			const linkPromises = urls.map(async (url) => {
+				try {
+					const response = await fetch("/api/onboarding/extract-content", {
+						method: "POST",
+						headers: { "Content-Type": "application/json" },
+						body: JSON.stringify({ urls: [url] }),
+					})
+					const data = await response.json()
+					return data.results?.[0] || null
+				} catch {
+					return null
+				} finally {
+					// Clear this URL's processing state
+					if (draftRequestIdRef.current === requestId) {
+						setProcessingByUrl((prev) => ({ ...prev, [url]: false }))
+					}
+				}
+			})
+
+			// Fetch X/Twitter research
+			const xResearchPromise = formData.twitter
+				? (async () => {
+						try {
+							const response = await fetch("/api/onboarding/research", {
+								method: "POST",
+								headers: { "Content-Type": "application/json" },
+								body: JSON.stringify({
+									xUrl: formData.twitter,
+									name: user?.name,
+									email: user?.email,
+								}),
+							})
+							if (!response.ok) return null
+							const data = await response.json()
+							return data?.text?.trim() || null
+						} catch {
+							return null
+						} finally {
+							// Clear twitter URL's processing state
+							if (draftRequestIdRef.current === requestId) {
+								setProcessingByUrl((prev) => ({
+									...prev,
+									[formData.twitter]: false,
+								}))
+							}
+						}
+					})()
+				: Promise.resolve(null)
 
 			const [exaResults, xResearchResult] = await Promise.all([
-				urls.length > 0
-					? fetch("/api/onboarding/extract-content", {
-							method: "POST",
-							headers: { "Content-Type": "application/json" },
-							body: JSON.stringify({ urls }),
-						})
-							.then((r) => r.json())
-							.then((data) => data.results || [])
-							.catch(() => [])
-					: Promise.resolve([]),
-				formData.twitter
-					? fetch("/api/onboarding/research", {
-							method: "POST",
-							headers: { "Content-Type": "application/json" },
-							body: JSON.stringify({
-								xUrl: formData.twitter,
-								name: user?.name,
-								email: user?.email,
-							}),
-						})
-							.then((r) => (r.ok ? r.json() : null))
-							.then((data) => data?.text?.trim() || null)
-							.catch(() => null)
-					: Promise.resolve(null),
+				Promise.all(linkPromises),
+				xResearchPromise,
 			])
 
+			// Guard against stale request completing after a newer one
+			if (draftRequestIdRef.current !== requestId) return
+
 			for (const result of exaResults) {
-				if (result.text || result.description) {
+				if (result && (result.text || result.description)) {
 					drafts.push({
 						kind: "link",
 						content: result.text || result.description || "",
@@ -304,7 +361,9 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 		} catch (error) {
 			console.warn("Error building draft docs:", error)
 		} finally {
-			setIsFetchingDrafts(false)
+			if (draftRequestIdRef.current === requestId) {
+				setIsFetchingDrafts(false)
+			}
 		}
 	}, [formData, user])
 
@@ -502,18 +561,23 @@ export function ChatSidebar({ formData }: ChatSidebarProps) {
 										{msg.type === "formData" && (
 											<div className="bg-[#293952]/40 rounded-lg p-2 px-3 space-y-1 flex-1">
 												{msg.title && (
-													<h3
-														className="text-sm font-medium"
-														style={{
-															background:
-																"linear-gradient(90deg, #369BFD 0%, #36FDFD 30%, #36FDB5 100%)",
-															WebkitBackgroundClip: "text",
-															WebkitTextFillColor: "transparent",
-															backgroundClip: "text",
-														}}
-													>
-														{msg.title}
-													</h3>
+													<div className="flex items-center gap-2">
+														<h3
+															className="text-sm font-medium"
+															style={{
+																background:
+																	"linear-gradient(90deg, #369BFD 0%, #36FDFD 30%, #36FDB5 100%)",
+																WebkitBackgroundClip: "text",
+																WebkitTextFillColor: "transparent",
+																backgroundClip: "text",
+															}}
+														>
+															{msg.title}
+														</h3>
+														{msg.url && processingByUrl[msg.url] && (
+															<Loader2 className="h-3 w-3 animate-spin text-blue-400" />
+														)}
+													</div>
 												)}
 												{msg.url && (
 													<a
diff --git a/apps/web/components/new/onboarding/setup/integrations-step.tsx b/apps/web/components/new/onboarding/setup/integrations-step.tsx
index 03102b14..951f26c0 100644
--- a/apps/web/components/new/onboarding/setup/integrations-step.tsx
+++ b/apps/web/components/new/onboarding/setup/integrations-step.tsx
@@ -7,7 +7,7 @@ import { XBookmarksDetailView } from "@/components/new/onboarding/x-bookmarks-de
 import { useRouter } from "next/navigation"
 import { cn } from "@lib/utils"
 import { dmSansClassName } from "@/lib/fonts"
-import { useOnboardingStorage } from "@hooks/use-onboarding-storage"
+import { useOrgOnboarding } from "@hooks/use-org-onboarding"
 import { analytics } from "@/lib/analytics"
 
 const integrationCards = [
@@ -61,11 +61,11 @@ const integrationCards = [
 export function IntegrationsStep() {
 	const router = useRouter()
 	const [selectedCard, setSelectedCard] = useState<string | null>(null)
-	const { markOnboardingCompleted } = useOnboardingStorage()
+	const { markOrgOnboarded } = useOrgOnboarding()
 
 	const handleContinue = () => {
+		markOrgOnboarded()
 		analytics.onboardingCompleted()
-		markOnboardingCompleted()
 		router.push("/new")
 	}