diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/build-geo.js | 108 | ||||
| -rw-r--r-- | scripts/build-prisma-client.js | 18 | ||||
| -rw-r--r-- | scripts/check-db.js | 90 | ||||
| -rw-r--r-- | scripts/check-env.js | 27 | ||||
| -rw-r--r-- | scripts/download-country-names.js | 59 | ||||
| -rw-r--r-- | scripts/download-language-names.js | 59 | ||||
| -rw-r--r-- | scripts/format-lang.js | 35 | ||||
| -rw-r--r-- | scripts/merge-messages.js | 43 | ||||
| -rw-r--r-- | scripts/postbuild.js | 10 | ||||
| -rw-r--r-- | scripts/seed-data.ts | 121 | ||||
| -rw-r--r-- | scripts/seed/distributions/devices.ts | 80 | ||||
| -rw-r--r-- | scripts/seed/distributions/geographic.ts | 144 | ||||
| -rw-r--r-- | scripts/seed/distributions/referrers.ts | 163 | ||||
| -rw-r--r-- | scripts/seed/distributions/temporal.ts | 69 | ||||
| -rw-r--r-- | scripts/seed/generators/events.ts | 191 | ||||
| -rw-r--r-- | scripts/seed/generators/revenue.ts | 65 | ||||
| -rw-r--r-- | scripts/seed/generators/sessions.ts | 52 | ||||
| -rw-r--r-- | scripts/seed/index.ts | 378 | ||||
| -rw-r--r-- | scripts/seed/sites/blog.ts | 108 | ||||
| -rw-r--r-- | scripts/seed/sites/saas.ts | 185 | ||||
| -rw-r--r-- | scripts/seed/utils.ts | 85 | ||||
| -rw-r--r-- | scripts/start-env.js | 7 | ||||
| -rw-r--r-- | scripts/telemetry.js | 40 | ||||
| -rw-r--r-- | scripts/update-tracker.js | 16 |
24 files changed, 2153 insertions, 0 deletions
diff --git a/scripts/build-geo.js b/scripts/build-geo.js new file mode 100644 index 0000000..a83caa6 --- /dev/null +++ b/scripts/build-geo.js @@ -0,0 +1,108 @@ +/* eslint-disable no-console */ +import 'dotenv/config'; +import fs from 'node:fs'; +import path from 'node:path'; +import https from 'https'; +import tar from 'tar'; +import zlib from 'zlib'; + +if (process.env.VERCEL && !process.env.BUILD_GEO) { + console.log('Vercel environment detected. Skipping geo setup.'); + process.exit(0); +} + +const db = 'GeoLite2-City'; + +// Support custom URL via environment variable +let url = process.env.GEO_DATABASE_URL; + +// Fallback to default URLs if not provided +if (!url) { + if (process.env.MAXMIND_LICENSE_KEY) { + url = + `https://download.maxmind.com/app/geoip_download` + + `?edition_id=${db}&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`; + } else { + url = `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${db}.tar.gz`; + } +} + +const dest = path.resolve(process.cwd(), 'geo'); + +if (!fs.existsSync(dest)) { + fs.mkdirSync(dest); +} + +// Check if URL points to a direct .mmdb file (already extracted) +const isDirectMmdb = url.endsWith('.mmdb'); + +// Download handler for compressed tar.gz files +const downloadCompressed = url => + new Promise(resolve => { + https.get(url, res => { + resolve(res.pipe(zlib.createGunzip({})).pipe(tar.t())); + }); + }); + +// Download handler for direct .mmdb files +const downloadDirect = (url, originalUrl) => + new Promise((resolve, reject) => { + https.get(url, res => { + // Follow redirects + if (res.statusCode === 301 || res.statusCode === 302) { + downloadDirect(res.headers.location, originalUrl || url) + .then(resolve) + .catch(reject); + return; + } + + const filename = path.join(dest, path.basename(originalUrl || url)); + const fileStream = fs.createWriteStream(filename); + + res.pipe(fileStream); + + fileStream.on('finish', () => { + fileStream.close(); + console.log('Saved geo database:', filename); + resolve(); + }); + + fileStream.on('error', e => { + reject(e); + }); + }); + }); + +// Execute download based on file type +if (isDirectMmdb) { + downloadDirect(url).catch(e => { + console.error('Failed to download geo database:', e); + process.exit(1); + }); +} else { + downloadCompressed(url) + .then( + res => + new Promise((resolve, reject) => { + res.on('entry', entry => { + if (entry.path.endsWith('.mmdb')) { + const filename = path.join(dest, path.basename(entry.path)); + entry.pipe(fs.createWriteStream(filename)); + + console.log('Saved geo database:', filename); + } + }); + + res.on('error', e => { + reject(e); + }); + res.on('finish', () => { + resolve(); + }); + }), + ) + .catch(e => { + console.error('Failed to download geo database:', e); + process.exit(1); + }); +} diff --git a/scripts/build-prisma-client.js b/scripts/build-prisma-client.js new file mode 100644 index 0000000..b5edc3d --- /dev/null +++ b/scripts/build-prisma-client.js @@ -0,0 +1,18 @@ +import esbuild from 'esbuild'; + +esbuild + .build({ + entryPoints: ['src/generated/prisma/client.ts'], // Adjust this to your entry file + bundle: true, // Bundle all files into one (optional) + outfile: 'generated/prisma/client.js', // Output file + platform: 'node', // For Node.js compatibility + target: 'es2020', // Target version of Node.js + format: 'esm', // Use ESM format + sourcemap: true, // Optional: generates source maps for debugging + external: [ + '../src/generated/prisma', // exclude generated client + '@prisma/client', // just in case + '.prisma/client', + ], // Optional: Exclude external dependencies from bundling + }) + .catch(() => process.exit(1)); diff --git a/scripts/check-db.js b/scripts/check-db.js new file mode 100644 index 0000000..68374f6 --- /dev/null +++ b/scripts/check-db.js @@ -0,0 +1,90 @@ +/* eslint-disable no-console */ +import 'dotenv/config'; +import { execSync } from 'node:child_process'; +import { PrismaPg } from '@prisma/adapter-pg'; +import chalk from 'chalk'; +import semver from 'semver'; +import { PrismaClient } from '../generated/prisma/client.js'; + +const MIN_VERSION = '9.4.0'; + +if (process.env.SKIP_DB_CHECK) { + console.log('Skipping database check.'); + process.exit(0); +} + +const url = new URL(process.env.DATABASE_URL); + +const adapter = new PrismaPg( + { connectionString: url.toString() }, + { schema: url.searchParams.get('schema') }, +); + +const prisma = new PrismaClient({ adapter }); + +function success(msg) { + console.log(chalk.greenBright(`✓ ${msg}`)); +} + +function error(msg) { + console.log(chalk.redBright(`✗ ${msg}`)); +} + +async function checkEnv() { + if (!process.env.DATABASE_URL) { + throw new Error('DATABASE_URL is not defined.'); + } else { + success('DATABASE_URL is defined.'); + } + + if (process.env.REDIS_URL) { + success('REDIS_URL is defined.'); + } +} + +async function checkConnection() { + try { + await prisma.$connect(); + + success('Database connection successful.'); + } catch (e) { + throw new Error('Unable to connect to the database: ' + e.message); + } +} + +async function checkDatabaseVersion() { + const query = await prisma.$queryRaw`select version() as version`; + const version = semver.valid(semver.coerce(query[0].version)); + + if (semver.lt(version, MIN_VERSION)) { + throw new Error( + `Database version is not compatible. Please upgrade to ${MIN_VERSION} or greater.`, + ); + } + + success('Database version check successful.'); +} + +async function applyMigration() { + if (!process.env.SKIP_DB_MIGRATION) { + console.log(execSync('prisma migrate deploy').toString()); + + success('Database is up to date.'); + } +} + +(async () => { + let err = false; + for (const fn of [checkEnv, checkConnection, checkDatabaseVersion, applyMigration]) { + try { + await fn(); + } catch (e) { + error(e.message); + err = true; + } finally { + if (err) { + process.exit(1); + } + } + } +})(); diff --git a/scripts/check-env.js b/scripts/check-env.js new file mode 100644 index 0000000..79c0984 --- /dev/null +++ b/scripts/check-env.js @@ -0,0 +1,27 @@ +/* eslint-disable no-console */ +import 'dotenv/config'; + +function checkMissing(vars) { + const missing = vars.reduce((arr, key) => { + if (!process.env[key]) { + arr.push(key); + } + return arr; + }, []); + + if (missing.length) { + console.log(`The following environment variables are not defined:`); + for (const item of missing) { + console.log(' - ', item); + } + process.exit(1); + } +} + +if (!process.env.SKIP_DB_CHECK && !process.env.DATABASE_TYPE) { + checkMissing(['DATABASE_URL']); +} + +if (process.env.CLOUD_URL) { + checkMissing(['CLOUD_URL', 'CLICKHOUSE_URL', 'REDIS_URL']); +} diff --git a/scripts/download-country-names.js b/scripts/download-country-names.js new file mode 100644 index 0000000..937fb22 --- /dev/null +++ b/scripts/download-country-names.js @@ -0,0 +1,59 @@ +/* eslint-disable no-console */ + +import path from 'node:path'; +import chalk from 'chalk'; +import fs from 'fs-extra'; +import https from 'https'; + +const src = path.resolve(process.cwd(), 'src/lang'); +const dest = path.resolve(process.cwd(), 'public/intl/country'); +const files = fs.readdirSync(src); + +const getUrl = locale => + `https://raw.githubusercontent.com/umpirsky/country-list/master/data/${locale}/country.json`; + +const asyncForEach = async (array, callback) => { + for (let index = 0; index < array.length; index++) { + await callback(array[index], index, array); + } +}; + +const downloadFile = (url, filepath) => + new Promise(resolve => { + https + .get(url, res => { + if (res.statusCode === 200) { + const fileStream = fs.createWriteStream(filepath); + res.pipe(fileStream); + fileStream.on('finish', () => { + fileStream.close(); + console.log('Downloaded', chalk.greenBright('->'), filepath); + resolve(); + }); + } else { + res.resume(); + console.warn(`Warning: ${url} returned ${res.statusCode}`); + resolve(); + } + }) + .on('error', err => { + console.error(`Error downloading ${url}:`, err.message); + resolve(); + }); + }); + +const download = async files => { + await fs.ensureDir(dest); + + await asyncForEach(files, async file => { + const locale = file.replace('-', '_').replace('.json', ''); + + const filename = path.join(dest, file); + if (!fs.existsSync(filename)) { + const url = getUrl(locale); + await downloadFile(url, filename); + } + }); +}; + +download(files); diff --git a/scripts/download-language-names.js b/scripts/download-language-names.js new file mode 100644 index 0000000..d3db601 --- /dev/null +++ b/scripts/download-language-names.js @@ -0,0 +1,59 @@ +/* eslint-disable no-console */ + +import path from 'node:path'; +import chalk from 'chalk'; +import fs from 'fs-extra'; +import https from 'https'; + +const src = path.resolve(process.cwd(), 'src/lang'); +const dest = path.resolve(process.cwd(), 'public/intl/language'); +const files = fs.readdirSync(src); + +const getUrl = locale => + `https://raw.githubusercontent.com/umpirsky/language-list/master/data/${locale}/language.json`; + +const asyncForEach = async (array, callback) => { + for (let index = 0; index < array.length; index++) { + await callback(array[index], index, array); + } +}; + +const downloadFile = (url, filepath) => + new Promise(resolve => { + https + .get(url, res => { + if (res.statusCode === 200) { + const fileStream = fs.createWriteStream(filepath); + res.pipe(fileStream); + fileStream.on('finish', () => { + fileStream.close(); + console.log('Downloaded', chalk.greenBright('->'), filepath); + resolve(); + }); + } else { + res.resume(); + console.warn(`Warning: ${url} returned ${res.statusCode}`); + resolve(); + } + }) + .on('error', err => { + console.error(`Error downloading ${url}:`, err.message); + resolve(); + }); + }); + +const download = async files => { + await fs.ensureDir(dest); + + await asyncForEach(files, async file => { + const locale = file.replace('-', '_').replace('.json', ''); + + const filename = path.join(dest, file); + if (!fs.existsSync(filename)) { + const url = getUrl(locale); + await downloadFile(url, filename); + } + }); +}; + +download(files); diff --git a/scripts/format-lang.js b/scripts/format-lang.js new file mode 100644 index 0000000..95c390e --- /dev/null +++ b/scripts/format-lang.js @@ -0,0 +1,35 @@ +import path from 'node:path'; +import del from 'del'; +import fs from 'fs-extra'; +import { createRequire } from 'module'; + +const require = createRequire(import.meta.url); +const src = path.resolve(process.cwd(), 'src/lang'); +const dest = path.resolve(process.cwd(), 'build/messages'); +const files = fs.readdirSync(src); + +del.sync([path.join(dest)]); + +/* +This script takes the files from the `lang` folder and formats them into +the format that format-js expects. + */ +async function run() { + await fs.ensureDir(dest); + + files.forEach(file => { + const lang = require(path.resolve(process.cwd(), `src/lang/${file}`)); + const keys = Object.keys(lang).sort(); + + const formatted = keys.reduce((obj, key) => { + obj[key] = { defaultMessage: lang[key] }; + return obj; + }, {}); + + const json = JSON.stringify(formatted, null, 2); + + fs.writeFileSync(path.resolve(dest, file), json); + }); +} + +run(); diff --git a/scripts/merge-messages.js b/scripts/merge-messages.js new file mode 100644 index 0000000..29abc53 --- /dev/null +++ b/scripts/merge-messages.js @@ -0,0 +1,43 @@ +/* eslint-disable no-console */ +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'module'; +import prettier from 'prettier'; + +const require = createRequire(import.meta.url); + +const messages = require('../build/extracted-messages.json'); +const dest = path.resolve(process.cwd(), 'src/lang'); +const files = fs.readdirSync(dest); +const keys = Object.keys(messages).sort(); + +/* +This script takes extracted messages and merges them +with the existing files under `lang`. Any newly added +keys will be printed to the console. + */ +files.forEach(file => { + const lang = require(path.resolve(process.cwd(), `src/lang/${file}`)); + + console.log(`Merging ${file}`); + + const merged = keys.reduce((obj, key) => { + const message = lang[key]; + + if (file === 'en-US.json') { + obj[key] = messages[key].defaultMessage; + } else { + obj[key] = message || messages[key].defaultMessage; + } + + if (!message) { + console.log(`* Added key ${key}`); + } + + return obj; + }, {}); + + const json = prettier.format(JSON.stringify(merged), { parser: 'json' }); + + fs.writeFileSync(path.resolve(dest, file), json); +}); diff --git a/scripts/postbuild.js b/scripts/postbuild.js new file mode 100644 index 0000000..2a4404c --- /dev/null +++ b/scripts/postbuild.js @@ -0,0 +1,10 @@ +import 'dotenv/config'; +import { sendTelemetry } from './telemetry.js'; + +async function run() { + if (!process.env.DISABLE_TELEMETRY) { + await sendTelemetry('build'); + } +} + +run(); diff --git a/scripts/seed-data.ts b/scripts/seed-data.ts new file mode 100644 index 0000000..82a0564 --- /dev/null +++ b/scripts/seed-data.ts @@ -0,0 +1,121 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ + +/** + * Umami Sample Data Generator + * + * Generates realistic analytics data for local development and testing. + * Creates two demo websites: + * - Demo Blog: Low traffic (~100 sessions/month) + * - Demo SaaS: Average traffic (~500 sessions/day) + * + * Usage: + * npm run seed-data # Generate 30 days of data + * npm run seed-data -- --days 90 # Generate 90 days of data + * npm run seed-data -- --clear # Clear existing demo data first + * npm run seed-data -- --verbose # Show detailed progress + */ + +import { seed, type SeedConfig } from './seed/index.js'; + +function parseArgs(): SeedConfig { + const args = process.argv.slice(2); + + const config: SeedConfig = { + days: 30, + clear: false, + verbose: false, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === '--days' && args[i + 1]) { + config.days = parseInt(args[i + 1], 10); + if (isNaN(config.days) || config.days < 1) { + console.error('Error: --days must be a positive integer'); + process.exit(1); + } + i++; + } else if (arg === '--clear') { + config.clear = true; + } else if (arg === '--verbose' || arg === '-v') { + config.verbose = true; + } else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } else if (arg.startsWith('--days=')) { + config.days = parseInt(arg.split('=')[1], 10); + if (isNaN(config.days) || config.days < 1) { + console.error('Error: --days must be a positive integer'); + process.exit(1); + } + } + } + + return config; +} + +function printHelp(): void { + console.log(` +Umami Sample Data Generator + +Generates realistic analytics data for local development and testing. + +Usage: + npm run seed-data [options] + +Options: + --days <number> Number of days of data to generate (default: 30) + --clear Clear existing demo data before generating + --verbose, -v Show detailed progress + --help, -h Show this help message + +Examples: + npm run seed-data # Generate 30 days of data + npm run seed-data -- --days 90 # Generate 90 days of data + npm run seed-data -- --clear # Clear existing demo data first + npm run seed-data -- --days 7 -v # Generate 7 days with verbose output + +Generated Sites: + - Demo Blog: Low traffic (~90 sessions/month) + - Demo SaaS: Average traffic (~500 sessions/day) with revenue tracking + +Note: + This script is blocked from running in production environments + (NODE_ENV=production or cloud platforms like Vercel/Netlify/Railway). +`); +} + +function checkEnvironment(): void { + const nodeEnv = process.env.NODE_ENV; + + if (nodeEnv === 'production') { + console.error('\nError: seed-data cannot run in production environment.'); + console.error('This script is only for local development and testing.\n'); + process.exit(1); + } + + if (process.env.VERCEL || process.env.NETLIFY || process.env.RAILWAY_ENVIRONMENT) { + console.error('\nError: seed-data cannot run in cloud environments.'); + console.error('This script is only for local development and testing.\n'); + process.exit(1); + } +} + +async function main(): Promise<void> { + console.log('\nUmami Sample Data Generator\n'); + + checkEnvironment(); + + const config = parseArgs(); + + try { + await seed(config); + } catch (error) { + console.error('\nError generating seed data:', error); + process.exit(1); + } +} + +main(); diff --git a/scripts/seed/distributions/devices.ts b/scripts/seed/distributions/devices.ts new file mode 100644 index 0000000..9d8b8c0 --- /dev/null +++ b/scripts/seed/distributions/devices.ts @@ -0,0 +1,80 @@ +import { weightedRandom, pickRandom, type WeightedOption } from '../utils.js'; + +export type DeviceType = 'desktop' | 'mobile' | 'tablet'; + +const deviceWeights: WeightedOption<DeviceType>[] = [ + { value: 'desktop', weight: 0.55 }, + { value: 'mobile', weight: 0.4 }, + { value: 'tablet', weight: 0.05 }, +]; + +const browsersByDevice: Record<DeviceType, WeightedOption<string>[]> = { + desktop: [ + { value: 'Chrome', weight: 0.65 }, + { value: 'Safari', weight: 0.12 }, + { value: 'Firefox', weight: 0.1 }, + { value: 'Edge', weight: 0.1 }, + { value: 'Opera', weight: 0.03 }, + ], + mobile: [ + { value: 'Chrome', weight: 0.55 }, + { value: 'Safari', weight: 0.35 }, + { value: 'Samsung', weight: 0.05 }, + { value: 'Firefox', weight: 0.03 }, + { value: 'Opera', weight: 0.02 }, + ], + tablet: [ + { value: 'Safari', weight: 0.6 }, + { value: 'Chrome', weight: 0.35 }, + { value: 'Firefox', weight: 0.05 }, + ], +}; + +const osByDevice: Record<DeviceType, WeightedOption<string>[]> = { + desktop: [ + { value: 'Windows 10', weight: 0.5 }, + { value: 'Mac OS', weight: 0.3 }, + { value: 'Linux', weight: 0.12 }, + { value: 'Chrome OS', weight: 0.05 }, + { value: 'Windows 11', weight: 0.03 }, + ], + mobile: [ + { value: 'iOS', weight: 0.45 }, + { value: 'Android', weight: 0.55 }, + ], + tablet: [ + { value: 'iOS', weight: 0.75 }, + { value: 'Android', weight: 0.25 }, + ], +}; + +const screensByDevice: Record<DeviceType, string[]> = { + desktop: [ + '1920x1080', + '2560x1440', + '1366x768', + '1440x900', + '3840x2160', + '1536x864', + '1680x1050', + '2560x1080', + ], + mobile: ['390x844', '414x896', '375x812', '360x800', '428x926', '393x873', '412x915', '360x780'], + tablet: ['1024x768', '768x1024', '834x1194', '820x1180', '810x1080', '800x1280'], +}; + +export interface DeviceInfo { + device: DeviceType; + browser: string; + os: string; + screen: string; +} + +export function getRandomDevice(): DeviceInfo { + const device = weightedRandom(deviceWeights); + const browser = weightedRandom(browsersByDevice[device]); + const os = weightedRandom(osByDevice[device]); + const screen = pickRandom(screensByDevice[device]); + + return { device, browser, os, screen }; +} diff --git a/scripts/seed/distributions/geographic.ts b/scripts/seed/distributions/geographic.ts new file mode 100644 index 0000000..ba6ebae --- /dev/null +++ b/scripts/seed/distributions/geographic.ts @@ -0,0 +1,144 @@ +import { weightedRandom, pickRandom, type WeightedOption } from '../utils.js'; + +interface GeoLocation { + country: string; + region: string; + city: string; +} + +const countryWeights: WeightedOption<string>[] = [ + { value: 'US', weight: 0.35 }, + { value: 'GB', weight: 0.08 }, + { value: 'DE', weight: 0.06 }, + { value: 'FR', weight: 0.05 }, + { value: 'CA', weight: 0.04 }, + { value: 'AU', weight: 0.03 }, + { value: 'IN', weight: 0.08 }, + { value: 'BR', weight: 0.04 }, + { value: 'JP', weight: 0.03 }, + { value: 'NL', weight: 0.02 }, + { value: 'ES', weight: 0.02 }, + { value: 'IT', weight: 0.02 }, + { value: 'PL', weight: 0.02 }, + { value: 'SE', weight: 0.01 }, + { value: 'MX', weight: 0.02 }, + { value: 'KR', weight: 0.02 }, + { value: 'SG', weight: 0.01 }, + { value: 'ID', weight: 0.02 }, + { value: 'PH', weight: 0.01 }, + { value: 'TH', weight: 0.01 }, + { value: 'VN', weight: 0.01 }, + { value: 'RU', weight: 0.02 }, + { value: 'UA', weight: 0.01 }, + { value: 'ZA', weight: 0.01 }, + { value: 'NG', weight: 0.01 }, +]; + +const regionsByCountry: Record<string, { region: string; city: string }[]> = { + US: [ + { region: 'CA', city: 'San Francisco' }, + { region: 'CA', city: 'Los Angeles' }, + { region: 'NY', city: 'New York' }, + { region: 'TX', city: 'Austin' }, + { region: 'TX', city: 'Houston' }, + { region: 'WA', city: 'Seattle' }, + { region: 'IL', city: 'Chicago' }, + { region: 'MA', city: 'Boston' }, + { region: 'CO', city: 'Denver' }, + { region: 'GA', city: 'Atlanta' }, + { region: 'FL', city: 'Miami' }, + { region: 'PA', city: 'Philadelphia' }, + ], + GB: [ + { region: 'ENG', city: 'London' }, + { region: 'ENG', city: 'Manchester' }, + { region: 'ENG', city: 'Birmingham' }, + { region: 'SCT', city: 'Edinburgh' }, + { region: 'ENG', city: 'Bristol' }, + ], + DE: [ + { region: 'BE', city: 'Berlin' }, + { region: 'BY', city: 'Munich' }, + { region: 'HH', city: 'Hamburg' }, + { region: 'HE', city: 'Frankfurt' }, + { region: 'NW', city: 'Cologne' }, + ], + FR: [ + { region: 'IDF', city: 'Paris' }, + { region: 'ARA', city: 'Lyon' }, + { region: 'PAC', city: 'Marseille' }, + { region: 'OCC', city: 'Toulouse' }, + ], + CA: [ + { region: 'ON', city: 'Toronto' }, + { region: 'BC', city: 'Vancouver' }, + { region: 'QC', city: 'Montreal' }, + { region: 'AB', city: 'Calgary' }, + ], + AU: [ + { region: 'NSW', city: 'Sydney' }, + { region: 'VIC', city: 'Melbourne' }, + { region: 'QLD', city: 'Brisbane' }, + { region: 'WA', city: 'Perth' }, + ], + IN: [ + { region: 'MH', city: 'Mumbai' }, + { region: 'KA', city: 'Bangalore' }, + { region: 'DL', city: 'New Delhi' }, + { region: 'TN', city: 'Chennai' }, + { region: 'TG', city: 'Hyderabad' }, + ], + BR: [ + { region: 'SP', city: 'Sao Paulo' }, + { region: 'RJ', city: 'Rio de Janeiro' }, + { region: 'MG', city: 'Belo Horizonte' }, + ], + JP: [ + { region: '13', city: 'Tokyo' }, + { region: '27', city: 'Osaka' }, + { region: '23', city: 'Nagoya' }, + ], + NL: [ + { region: 'NH', city: 'Amsterdam' }, + { region: 'ZH', city: 'Rotterdam' }, + { region: 'ZH', city: 'The Hague' }, + ], +}; + +const defaultRegions = [{ region: '', city: '' }]; + +export function getRandomGeo(): GeoLocation { + const country = weightedRandom(countryWeights); + const regions = regionsByCountry[country] || defaultRegions; + const { region, city } = pickRandom(regions); + + return { country, region, city }; +} + +const languages: WeightedOption<string>[] = [ + { value: 'en-US', weight: 0.4 }, + { value: 'en-GB', weight: 0.08 }, + { value: 'de-DE', weight: 0.06 }, + { value: 'fr-FR', weight: 0.05 }, + { value: 'es-ES', weight: 0.05 }, + { value: 'pt-BR', weight: 0.04 }, + { value: 'ja-JP', weight: 0.03 }, + { value: 'zh-CN', weight: 0.05 }, + { value: 'ko-KR', weight: 0.02 }, + { value: 'ru-RU', weight: 0.02 }, + { value: 'it-IT', weight: 0.02 }, + { value: 'nl-NL', weight: 0.02 }, + { value: 'pl-PL', weight: 0.02 }, + { value: 'hi-IN', weight: 0.04 }, + { value: 'ar-SA', weight: 0.02 }, + { value: 'tr-TR', weight: 0.02 }, + { value: 'vi-VN', weight: 0.01 }, + { value: 'th-TH', weight: 0.01 }, + { value: 'id-ID', weight: 0.02 }, + { value: 'sv-SE', weight: 0.01 }, + { value: 'da-DK', weight: 0.01 }, +]; + +export function getRandomLanguage(): string { + return weightedRandom(languages); +} diff --git a/scripts/seed/distributions/referrers.ts b/scripts/seed/distributions/referrers.ts new file mode 100644 index 0000000..5b3f2c4 --- /dev/null +++ b/scripts/seed/distributions/referrers.ts @@ -0,0 +1,163 @@ +import { weightedRandom, pickRandom, randomInt, type WeightedOption } from '../utils.js'; + +export type ReferrerType = 'direct' | 'organic' | 'social' | 'paid' | 'referral'; + +export interface ReferrerInfo { + type: ReferrerType; + domain: string | null; + path: string | null; + utmSource: string | null; + utmMedium: string | null; + utmCampaign: string | null; + utmContent: string | null; + utmTerm: string | null; + gclid: string | null; + fbclid: string | null; +} + +const referrerTypeWeights: WeightedOption<ReferrerType>[] = [ + { value: 'direct', weight: 0.4 }, + { value: 'organic', weight: 0.25 }, + { value: 'social', weight: 0.15 }, + { value: 'paid', weight: 0.1 }, + { value: 'referral', weight: 0.1 }, +]; + +const searchEngines = [ + { domain: 'google.com', path: '/search' }, + { domain: 'bing.com', path: '/search' }, + { domain: 'duckduckgo.com', path: '/' }, + { domain: 'yahoo.com', path: '/search' }, + { domain: 'baidu.com', path: '/s' }, +]; + +const socialPlatforms = [ + { domain: 'twitter.com', path: null }, + { domain: 'x.com', path: null }, + { domain: 'linkedin.com', path: '/feed' }, + { domain: 'facebook.com', path: null }, + { domain: 'reddit.com', path: '/r/programming' }, + { domain: 'news.ycombinator.com', path: '/item' }, + { domain: 'threads.net', path: null }, + { domain: 'bsky.app', path: null }, +]; + +const referralSites = [ + { domain: 'medium.com', path: '/@author/article' }, + { domain: 'dev.to', path: '/post' }, + { domain: 'hashnode.com', path: '/blog' }, + { domain: 'techcrunch.com', path: '/article' }, + { domain: 'producthunt.com', path: '/posts' }, + { domain: 'indiehackers.com', path: '/post' }, +]; + +interface PaidCampaign { + source: string; + medium: string; + campaign: string; + useGclid?: boolean; + useFbclid?: boolean; +} + +const paidCampaigns: PaidCampaign[] = [ + { source: 'google', medium: 'cpc', campaign: 'brand_search', useGclid: true }, + { source: 'google', medium: 'cpc', campaign: 'product_awareness', useGclid: true }, + { source: 'facebook', medium: 'paid_social', campaign: 'retargeting', useFbclid: true }, + { source: 'facebook', medium: 'paid_social', campaign: 'lookalike', useFbclid: true }, + { source: 'linkedin', medium: 'cpc', campaign: 'b2b_targeting' }, + { source: 'twitter', medium: 'paid_social', campaign: 'launch_promo' }, +]; + +const organicCampaigns = [ + { source: 'newsletter', medium: 'email', campaign: 'weekly_digest' }, + { source: 'newsletter', medium: 'email', campaign: 'product_update' }, + { source: 'partner', medium: 'referral', campaign: 'integration_launch' }, +]; + +function generateClickId(): string { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + for (let i = 0; i < 32; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} + +export function getRandomReferrer(): ReferrerInfo { + const type = weightedRandom(referrerTypeWeights); + + const result: ReferrerInfo = { + type, + domain: null, + path: null, + utmSource: null, + utmMedium: null, + utmCampaign: null, + utmContent: null, + utmTerm: null, + gclid: null, + fbclid: null, + }; + + switch (type) { + case 'direct': + // No referrer data + break; + + case 'organic': { + const engine = pickRandom(searchEngines); + result.domain = engine.domain; + result.path = engine.path; + break; + } + + case 'social': { + const platform = pickRandom(socialPlatforms); + result.domain = platform.domain; + result.path = platform.path; + + // Some social traffic has UTM params + if (Math.random() < 0.3) { + result.utmSource = platform.domain.replace('.com', '').replace('.net', ''); + result.utmMedium = 'social'; + } + break; + } + + case 'paid': { + const campaign = pickRandom(paidCampaigns); + result.utmSource = campaign.source; + result.utmMedium = campaign.medium; + result.utmCampaign = campaign.campaign; + result.utmContent = `ad_${randomInt(1, 5)}`; + + if (campaign.useGclid) { + result.gclid = generateClickId(); + result.domain = 'google.com'; + result.path = '/search'; + } else if (campaign.useFbclid) { + result.fbclid = generateClickId(); + result.domain = 'facebook.com'; + result.path = null; + } + break; + } + + case 'referral': { + // Mix of pure referrals and organic campaigns + if (Math.random() < 0.6) { + const site = pickRandom(referralSites); + result.domain = site.domain; + result.path = site.path; + } else { + const campaign = pickRandom(organicCampaigns); + result.utmSource = campaign.source; + result.utmMedium = campaign.medium; + result.utmCampaign = campaign.campaign; + } + break; + } + } + + return result; +} diff --git a/scripts/seed/distributions/temporal.ts b/scripts/seed/distributions/temporal.ts new file mode 100644 index 0000000..da0409a --- /dev/null +++ b/scripts/seed/distributions/temporal.ts @@ -0,0 +1,69 @@ +import { weightedRandom, randomInt, type WeightedOption } from '../utils.js'; + +const hourlyWeights: WeightedOption<number>[] = [ + { value: 0, weight: 0.02 }, + { value: 1, weight: 0.01 }, + { value: 2, weight: 0.01 }, + { value: 3, weight: 0.01 }, + { value: 4, weight: 0.01 }, + { value: 5, weight: 0.02 }, + { value: 6, weight: 0.03 }, + { value: 7, weight: 0.05 }, + { value: 8, weight: 0.07 }, + { value: 9, weight: 0.08 }, + { value: 10, weight: 0.09 }, + { value: 11, weight: 0.08 }, + { value: 12, weight: 0.07 }, + { value: 13, weight: 0.08 }, + { value: 14, weight: 0.09 }, + { value: 15, weight: 0.08 }, + { value: 16, weight: 0.07 }, + { value: 17, weight: 0.06 }, + { value: 18, weight: 0.05 }, + { value: 19, weight: 0.04 }, + { value: 20, weight: 0.03 }, + { value: 21, weight: 0.03 }, + { value: 22, weight: 0.02 }, + { value: 23, weight: 0.02 }, +]; + +const dayOfWeekWeights: WeightedOption<number>[] = [ + { value: 0, weight: 0.08 }, // Sunday + { value: 1, weight: 0.16 }, // Monday + { value: 2, weight: 0.17 }, // Tuesday + { value: 3, weight: 0.17 }, // Wednesday + { value: 4, weight: 0.16 }, // Thursday + { value: 5, weight: 0.15 }, // Friday + { value: 6, weight: 0.11 }, // Saturday +]; + +export function getWeightedHour(): number { + return weightedRandom(hourlyWeights); +} + +export function getDayOfWeekMultiplier(dayOfWeek: number): number { + const weight = dayOfWeekWeights.find(d => d.value === dayOfWeek)?.weight ?? 0.14; + return weight / 0.14; // Normalize around 1.0 +} + +export function generateTimestampForDay(day: Date): Date { + const hour = getWeightedHour(); + const minute = randomInt(0, 59); + const second = randomInt(0, 59); + const millisecond = randomInt(0, 999); + + const timestamp = new Date(day); + timestamp.setHours(hour, minute, second, millisecond); + + return timestamp; +} + +export function getSessionCountForDay(baseCount: number, day: Date): number { + const dayOfWeek = day.getDay(); + const multiplier = getDayOfWeekMultiplier(dayOfWeek); + + // Add some random variance (±20%) + const variance = 0.8 + Math.random() * 0.4; + + return Math.round(baseCount * multiplier * variance); +} diff --git a/scripts/seed/generators/events.ts b/scripts/seed/generators/events.ts new file mode 100644 index 0000000..7242906 --- /dev/null +++ b/scripts/seed/generators/events.ts @@ -0,0 +1,191 @@ +import { uuid, addSeconds, randomInt } from '../utils.js'; +import { getRandomReferrer } from '../distributions/referrers.js'; +import type { SessionData } from './sessions.js'; + +export const EVENT_TYPE = { + pageView: 1, + customEvent: 2, +} as const; + +export interface PageConfig { + path: string; + title: string; + weight: number; + avgTimeOnPage: number; +} + +export interface CustomEventConfig { + name: string; + weight: number; + pages?: string[]; + data?: Record<string, string[] | number[]>; +} + +export interface JourneyConfig { + pages: string[]; + weight: number; +} + +export interface EventData { + id: string; + websiteId: string; + sessionId: string; + visitId: string; + eventType: number; + urlPath: string; + urlQuery: string | null; + pageTitle: string | null; + hostname: string; + referrerDomain: string | null; + referrerPath: string | null; + utmSource: string | null; + utmMedium: string | null; + utmCampaign: string | null; + utmContent: string | null; + utmTerm: string | null; + gclid: string | null; + fbclid: string | null; + eventName: string | null; + tag: string | null; + createdAt: Date; +} + +export interface EventDataEntry { + id: string; + websiteId: string; + websiteEventId: string; + dataKey: string; + stringValue: string | null; + numberValue: number | null; + dateValue: Date | null; + dataType: number; + createdAt: Date; +} + +export interface SiteConfig { + hostname: string; + pages: PageConfig[]; + journeys: JourneyConfig[]; + customEvents: CustomEventConfig[]; +} + +function getPageTitle(pages: PageConfig[], path: string): string | null { + const page = pages.find(p => p.path === path); + return page?.title ?? null; +} + +function getPageTimeOnPage(pages: PageConfig[], path: string): number { + const page = pages.find(p => p.path === path); + return page?.avgTimeOnPage ?? 30; +} + +export function generateEventsForSession( + session: SessionData, + siteConfig: SiteConfig, + journey: string[], +): { events: EventData[]; eventDataEntries: EventDataEntry[] } { + const events: EventData[] = []; + const eventDataEntries: EventDataEntry[] = []; + const visitId = uuid(); + + let currentTime = session.createdAt; + const referrer = getRandomReferrer(); + + for (let i = 0; i < journey.length; i++) { + const pagePath = journey[i]; + const isFirstPage = i === 0; + + const eventId = uuid(); + const pageTitle = getPageTitle(siteConfig.pages, pagePath); + + events.push({ + id: eventId, + websiteId: session.websiteId, + sessionId: session.id, + visitId, + eventType: EVENT_TYPE.pageView, + urlPath: pagePath, + urlQuery: null, + pageTitle, + hostname: siteConfig.hostname, + referrerDomain: isFirstPage ? referrer.domain : null, + referrerPath: isFirstPage ? referrer.path : null, + utmSource: isFirstPage ? referrer.utmSource : null, + utmMedium: isFirstPage ? referrer.utmMedium : null, + utmCampaign: isFirstPage ? referrer.utmCampaign : null, + utmContent: isFirstPage ? referrer.utmContent : null, + utmTerm: isFirstPage ? referrer.utmTerm : null, + gclid: isFirstPage ? referrer.gclid : null, + fbclid: isFirstPage ? referrer.fbclid : null, + eventName: null, + tag: null, + createdAt: currentTime, + }); + + // Check for custom events on this page + for (const customEvent of siteConfig.customEvents) { + // Check if this event can occur on this page + if (customEvent.pages && !customEvent.pages.includes(pagePath)) { + continue; + } + + // Random chance based on weight + if (Math.random() < customEvent.weight) { + currentTime = addSeconds(currentTime, randomInt(2, 15)); + + const customEventId = uuid(); + events.push({ + id: customEventId, + websiteId: session.websiteId, + sessionId: session.id, + visitId, + eventType: EVENT_TYPE.customEvent, + urlPath: pagePath, + urlQuery: null, + pageTitle, + hostname: siteConfig.hostname, + referrerDomain: null, + referrerPath: null, + utmSource: null, + utmMedium: null, + utmCampaign: null, + utmContent: null, + utmTerm: null, + gclid: null, + fbclid: null, + eventName: customEvent.name, + tag: null, + createdAt: currentTime, + }); + + // Generate event data if configured + if (customEvent.data) { + for (const [key, values] of Object.entries(customEvent.data)) { + const value = values[Math.floor(Math.random() * values.length)]; + const isNumber = typeof value === 'number'; + + eventDataEntries.push({ + id: uuid(), + websiteId: session.websiteId, + websiteEventId: customEventId, + dataKey: key, + stringValue: isNumber ? null : String(value), + numberValue: isNumber ? value : null, + dateValue: null, + dataType: isNumber ? 2 : 1, // 1 = string, 2 = number + createdAt: currentTime, + }); + } + } + } + } + + // Time spent on page before navigating + const timeOnPage = getPageTimeOnPage(siteConfig.pages, pagePath); + const variance = Math.floor(timeOnPage * 0.5); + const actualTime = timeOnPage + randomInt(-variance, variance); + currentTime = addSeconds(currentTime, Math.max(5, actualTime)); + } + + return { events, eventDataEntries }; +} diff --git a/scripts/seed/generators/revenue.ts b/scripts/seed/generators/revenue.ts new file mode 100644 index 0000000..deea9e6 --- /dev/null +++ b/scripts/seed/generators/revenue.ts @@ -0,0 +1,65 @@ +import { uuid, randomFloat } from '../utils.js'; +import type { EventData } from './events.js'; + +export interface RevenueConfig { + eventName: string; + minAmount: number; + maxAmount: number; + currency: string; + weight: number; +} + +export interface RevenueData { + id: string; + websiteId: string; + sessionId: string; + eventId: string; + eventName: string; + currency: string; + revenue: number; + createdAt: Date; +} + +export function generateRevenue(event: EventData, config: RevenueConfig): RevenueData | null { + if (event.eventName !== config.eventName) { + return null; + } + + if (Math.random() > config.weight) { + return null; + } + + const revenue = randomFloat(config.minAmount, config.maxAmount); + + return { + id: uuid(), + websiteId: event.websiteId, + sessionId: event.sessionId, + eventId: event.id, + eventName: event.eventName!, + currency: config.currency, + revenue: Math.round(revenue * 100) / 100, // Round to 2 decimal places + createdAt: event.createdAt, + }; +} + +export function generateRevenueForEvents( + events: EventData[], + configs: RevenueConfig[], +): RevenueData[] { + const revenueEntries: RevenueData[] = []; + + for (const event of events) { + if (!event.eventName) continue; + + for (const config of configs) { + const revenue = generateRevenue(event, config); + if (revenue) { + revenueEntries.push(revenue); + break; // Only one revenue per event + } + } + } + + return revenueEntries; +} diff --git a/scripts/seed/generators/sessions.ts b/scripts/seed/generators/sessions.ts new file mode 100644 index 0000000..1370511 --- /dev/null +++ b/scripts/seed/generators/sessions.ts @@ -0,0 +1,52 @@ +import { uuid } from '../utils.js'; +import { getRandomDevice } from '../distributions/devices.js'; +import { getRandomGeo, getRandomLanguage } from '../distributions/geographic.js'; +import { generateTimestampForDay } from '../distributions/temporal.js'; + +export interface SessionData { + id: string; + websiteId: string; + browser: string; + os: string; + device: string; + screen: string; + language: string; + country: string; + region: string; + city: string; + createdAt: Date; +} + +export function createSession(websiteId: string, day: Date): SessionData { + const deviceInfo = getRandomDevice(); + const geo = getRandomGeo(); + const language = getRandomLanguage(); + const createdAt = generateTimestampForDay(day); + + return { + id: uuid(), + websiteId, + browser: deviceInfo.browser, + os: deviceInfo.os, + device: deviceInfo.device, + screen: deviceInfo.screen, + language, + country: geo.country, + region: geo.region, + city: geo.city, + createdAt, + }; +} + +export function createSessions(websiteId: string, day: Date, count: number): SessionData[] { + const sessions: SessionData[] = []; + + for (let i = 0; i < count; i++) { + sessions.push(createSession(websiteId, day)); + } + + // Sort by createdAt to maintain chronological order + sessions.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()); + + return sessions; +} diff --git a/scripts/seed/index.ts b/scripts/seed/index.ts new file mode 100644 index 0000000..5b9de8d --- /dev/null +++ b/scripts/seed/index.ts @@ -0,0 +1,378 @@ +/* eslint-disable no-console */ +import 'dotenv/config'; +import { PrismaPg } from '@prisma/adapter-pg'; +import { PrismaClient, Prisma } from '../../src/generated/prisma/client.js'; +import { uuid, generateDatesBetween, subDays, formatNumber, progressBar } from './utils.js'; +import { createSessions, type SessionData } from './generators/sessions.js'; +import { + generateEventsForSession, + type EventData, + type EventDataEntry, +} from './generators/events.js'; +import { + generateRevenueForEvents, + type RevenueData, + type RevenueConfig, +} from './generators/revenue.js'; +import { getSessionCountForDay } from './distributions/temporal.js'; +import { + BLOG_WEBSITE_NAME, + BLOG_WEBSITE_DOMAIN, + BLOG_SESSIONS_PER_DAY, + getBlogSiteConfig, + getBlogJourney, +} from './sites/blog.js'; +import { + SAAS_WEBSITE_NAME, + SAAS_WEBSITE_DOMAIN, + SAAS_SESSIONS_PER_DAY, + getSaasSiteConfig, + getSaasJourney, + saasRevenueConfigs, +} from './sites/saas.js'; + +const BATCH_SIZE = 1000; + +type SessionCreateInput = Prisma.SessionCreateManyInput; +type WebsiteEventCreateInput = Prisma.WebsiteEventCreateManyInput; +type EventDataCreateInput = Prisma.EventDataCreateManyInput; +type RevenueCreateInput = Prisma.RevenueCreateManyInput; + +export interface SeedConfig { + days: number; + clear: boolean; + verbose: boolean; +} + +export interface SeedResult { + websites: number; + sessions: number; + events: number; + eventData: number; + revenue: number; +} + +async function batchInsertSessions( + prisma: PrismaClient, + data: SessionCreateInput[], + verbose: boolean, +): Promise<void> { + for (let i = 0; i < data.length; i += BATCH_SIZE) { + const batch = data.slice(i, i + BATCH_SIZE); + await prisma.session.createMany({ data: batch, skipDuplicates: true }); + if (verbose) { + console.log( + ` Inserted ${Math.min(i + BATCH_SIZE, data.length)}/${data.length} session records`, + ); + } + } +} + +async function batchInsertEvents( + prisma: PrismaClient, + data: WebsiteEventCreateInput[], + verbose: boolean, +): Promise<void> { + for (let i = 0; i < data.length; i += BATCH_SIZE) { + const batch = data.slice(i, i + BATCH_SIZE); + await prisma.websiteEvent.createMany({ data: batch, skipDuplicates: true }); + if (verbose) { + console.log( + ` Inserted ${Math.min(i + BATCH_SIZE, data.length)}/${data.length} event records`, + ); + } + } +} + +async function batchInsertEventData( + prisma: PrismaClient, + data: EventDataCreateInput[], + verbose: boolean, +): Promise<void> { + for (let i = 0; i < data.length; i += BATCH_SIZE) { + const batch = data.slice(i, i + BATCH_SIZE); + await prisma.eventData.createMany({ data: batch, skipDuplicates: true }); + if (verbose) { + console.log( + ` Inserted ${Math.min(i + BATCH_SIZE, data.length)}/${data.length} eventData records`, + ); + } + } +} + +async function batchInsertRevenue( + prisma: PrismaClient, + data: RevenueCreateInput[], + verbose: boolean, +): Promise<void> { + for (let i = 0; i < data.length; i += BATCH_SIZE) { + const batch = data.slice(i, i + BATCH_SIZE); + await prisma.revenue.createMany({ data: batch, skipDuplicates: true }); + if (verbose) { + console.log( + ` Inserted ${Math.min(i + BATCH_SIZE, data.length)}/${data.length} revenue records`, + ); + } + } +} + +async function findAdminUser(prisma: PrismaClient): Promise<string> { + const adminUser = await prisma.user.findFirst({ + where: { role: 'admin' }, + select: { id: true }, + }); + + if (!adminUser) { + throw new Error( + 'No admin user found in the database.\n' + + 'Please ensure you have run the initial setup and created an admin user.\n' + + 'The default admin user is created during first build (username: admin, password: umami).', + ); + } + + return adminUser.id; +} + +async function createWebsite( + prisma: PrismaClient, + name: string, + domain: string, + adminUserId: string, +): Promise<string> { + const websiteId = uuid(); + + await prisma.website.create({ + data: { + id: websiteId, + name, + domain, + userId: adminUserId, + createdBy: adminUserId, + }, + }); + + return websiteId; +} + +async function clearDemoData(prisma: PrismaClient): Promise<void> { + console.log('Clearing existing demo data...'); + + const demoWebsites = await prisma.website.findMany({ + where: { + OR: [{ name: BLOG_WEBSITE_NAME }, { name: SAAS_WEBSITE_NAME }], + }, + select: { id: true }, + }); + + const websiteIds = demoWebsites.map(w => w.id); + + if (websiteIds.length === 0) { + console.log(' No existing demo websites found'); + return; + } + + console.log(` Found ${websiteIds.length} demo website(s)`); + + // Delete in correct order due to foreign key constraints + await prisma.revenue.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.eventData.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.sessionData.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.websiteEvent.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.session.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.segment.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.report.deleteMany({ where: { websiteId: { in: websiteIds } } }); + await prisma.website.deleteMany({ where: { id: { in: websiteIds } } }); + + console.log(' Cleared existing demo data'); +} + +interface SiteGeneratorConfig { + name: string; + domain: string; + sessionsPerDay: number; + getSiteConfig: () => ReturnType<typeof getBlogSiteConfig>; + getJourney: () => string[]; + revenueConfigs?: RevenueConfig[]; +} + +async function generateSiteData( + prisma: PrismaClient, + config: SiteGeneratorConfig, + days: Date[], + adminUserId: string, + verbose: boolean, +): Promise<{ sessions: number; events: number; eventData: number; revenue: number }> { + console.log(`\nGenerating data for ${config.name}...`); + + const websiteId = await createWebsite(prisma, config.name, config.domain, adminUserId); + console.log(` Created website: ${config.name} (${websiteId})`); + + const siteConfig = config.getSiteConfig(); + + const allSessions: SessionData[] = []; + const allEvents: EventData[] = []; + const allEventData: EventDataEntry[] = []; + const allRevenue: RevenueData[] = []; + + for (let dayIndex = 0; dayIndex < days.length; dayIndex++) { + const day = days[dayIndex]; + const sessionCount = getSessionCountForDay(config.sessionsPerDay, day); + const sessions = createSessions(websiteId, day, sessionCount); + + for (const session of sessions) { + const journey = config.getJourney(); + const { events, eventDataEntries } = generateEventsForSession(session, siteConfig, journey); + + allSessions.push(session); + allEvents.push(...events); + allEventData.push(...eventDataEntries); + + if (config.revenueConfigs) { + const revenueEntries = generateRevenueForEvents(events, config.revenueConfigs); + allRevenue.push(...revenueEntries); + } + } + + // Show progress (every day in verbose mode, otherwise every 2 days) + const shouldShowProgress = verbose || dayIndex % 2 === 0 || dayIndex === days.length - 1; + if (shouldShowProgress) { + process.stdout.write( + `\r ${progressBar(dayIndex + 1, days.length)} Day ${dayIndex + 1}/${days.length}`, + ); + } + } + + console.log(''); // New line after progress bar + + // Batch insert all data + console.log(` Inserting ${formatNumber(allSessions.length)} sessions...`); + await batchInsertSessions(prisma, allSessions as SessionCreateInput[], verbose); + + console.log(` Inserting ${formatNumber(allEvents.length)} events...`); + await batchInsertEvents(prisma, allEvents as WebsiteEventCreateInput[], verbose); + + if (allEventData.length > 0) { + console.log(` Inserting ${formatNumber(allEventData.length)} event data entries...`); + await batchInsertEventData(prisma, allEventData as EventDataCreateInput[], verbose); + } + + if (allRevenue.length > 0) { + console.log(` Inserting ${formatNumber(allRevenue.length)} revenue entries...`); + await batchInsertRevenue(prisma, allRevenue as RevenueCreateInput[], verbose); + } + + return { + sessions: allSessions.length, + events: allEvents.length, + eventData: allEventData.length, + revenue: allRevenue.length, + }; +} + +function createPrismaClient(): PrismaClient { + const url = process.env.DATABASE_URL; + if (!url) { + throw new Error( + 'DATABASE_URL environment variable is not set.\n' + + 'Please set DATABASE_URL in your .env file or environment.\n' + + 'Example: DATABASE_URL=postgresql://user:password@localhost:5432/umami', + ); + } + + let schema: string | undefined; + try { + const connectionUrl = new URL(url); + schema = connectionUrl.searchParams.get('schema') ?? undefined; + } catch { + throw new Error( + 'DATABASE_URL is not a valid URL.\n' + + 'Expected format: postgresql://user:password@host:port/database\n' + + `Received: ${url.substring(0, 30)}...`, + ); + } + + const adapter = new PrismaPg({ connectionString: url }, { schema }); + + return new PrismaClient({ + adapter, + errorFormat: 'pretty', + }); +} + +export async function seed(config: SeedConfig): Promise<SeedResult> { + const prisma = createPrismaClient(); + + try { + const endDate = new Date(); + const startDate = subDays(endDate, config.days); + const days = generateDatesBetween(startDate, endDate); + + console.log(`\nSeed Configuration:`); + console.log( + ` Date range: ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`, + ); + console.log(` Days: ${days.length}`); + console.log(` Clear existing: ${config.clear}`); + + if (config.clear) { + await clearDemoData(prisma); + } + + // Find admin user to own the demo websites + const adminUserId = await findAdminUser(prisma); + console.log(` Using admin user: ${adminUserId}`); + + // Generate Blog site (low traffic) + const blogResults = await generateSiteData( + prisma, + { + name: BLOG_WEBSITE_NAME, + domain: BLOG_WEBSITE_DOMAIN, + sessionsPerDay: BLOG_SESSIONS_PER_DAY, + getSiteConfig: getBlogSiteConfig, + getJourney: getBlogJourney, + }, + days, + adminUserId, + config.verbose, + ); + + // Generate SaaS site (high traffic) + const saasResults = await generateSiteData( + prisma, + { + name: SAAS_WEBSITE_NAME, + domain: SAAS_WEBSITE_DOMAIN, + sessionsPerDay: SAAS_SESSIONS_PER_DAY, + getSiteConfig: getSaasSiteConfig, + getJourney: getSaasJourney, + revenueConfigs: saasRevenueConfigs, + }, + days, + adminUserId, + config.verbose, + ); + + const result: SeedResult = { + websites: 2, + sessions: blogResults.sessions + saasResults.sessions, + events: blogResults.events + saasResults.events, + eventData: blogResults.eventData + saasResults.eventData, + revenue: blogResults.revenue + saasResults.revenue, + }; + + console.log(`\n${'─'.repeat(50)}`); + console.log(`Seed Complete!`); + console.log(`${'─'.repeat(50)}`); + console.log(` Websites: ${formatNumber(result.websites)}`); + console.log(` Sessions: ${formatNumber(result.sessions)}`); + console.log(` Events: ${formatNumber(result.events)}`); + console.log(` Event Data: ${formatNumber(result.eventData)}`); + console.log(` Revenue: ${formatNumber(result.revenue)}`); + console.log(`${'─'.repeat(50)}\n`); + + return result; + } finally { + await prisma.$disconnect(); + } +} diff --git a/scripts/seed/sites/blog.ts b/scripts/seed/sites/blog.ts new file mode 100644 index 0000000..e60b8b9 --- /dev/null +++ b/scripts/seed/sites/blog.ts @@ -0,0 +1,108 @@ +import { weightedRandom, type WeightedOption } from '../utils.js'; +import type { + SiteConfig, + JourneyConfig, + PageConfig, + CustomEventConfig, +} from '../generators/events.js'; + +export const BLOG_WEBSITE_NAME = 'Demo Blog'; +export const BLOG_WEBSITE_DOMAIN = 'blog.example.com'; + +const blogPosts = [ + 'getting-started-with-analytics', + 'privacy-first-tracking', + 'understanding-your-visitors', + 'improving-page-performance', + 'seo-best-practices', + 'content-marketing-guide', + 'building-audience-trust', + 'data-driven-decisions', +]; + +export const blogPages: PageConfig[] = [ + { path: '/', title: 'Demo Blog - Home', weight: 0.25, avgTimeOnPage: 30 }, + { path: '/blog', title: 'Blog Posts', weight: 0.2, avgTimeOnPage: 45 }, + { path: '/about', title: 'About Us', weight: 0.1, avgTimeOnPage: 60 }, + { path: '/contact', title: 'Contact', weight: 0.05, avgTimeOnPage: 45 }, + ...blogPosts.map(slug => ({ + path: `/blog/${slug}`, + title: slug + .split('-') + .map(w => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' '), + weight: 0.05, + avgTimeOnPage: 180, + })), +]; + +export const blogJourneys: JourneyConfig[] = [ + // Direct to blog post (organic search) + { pages: ['/blog/getting-started-with-analytics'], weight: 0.15 }, + { pages: ['/blog/privacy-first-tracking'], weight: 0.12 }, + { pages: ['/blog/understanding-your-visitors'], weight: 0.1 }, + + // Homepage bounces + { pages: ['/'], weight: 0.15 }, + + // Homepage to blog listing + { pages: ['/', '/blog'], weight: 0.1 }, + + // Homepage to blog post + { pages: ['/', '/blog', '/blog/seo-best-practices'], weight: 0.08 }, + { pages: ['/', '/blog', '/blog/content-marketing-guide'], weight: 0.08 }, + + // About page visits + { pages: ['/', '/about'], weight: 0.07 }, + { pages: ['/', '/about', '/contact'], weight: 0.05 }, + + // Blog post to another + { pages: ['/blog/improving-page-performance', '/blog/data-driven-decisions'], weight: 0.05 }, + + // Longer sessions + { pages: ['/', '/blog', '/blog/building-audience-trust', '/about'], weight: 0.05 }, +]; + +export const blogCustomEvents: CustomEventConfig[] = [ + { + name: 'newsletter_signup', + weight: 0.03, + pages: ['/', '/blog'], + }, + { + name: 'share_click', + weight: 0.05, + pages: blogPosts.map(slug => `/blog/${slug}`), + data: { + platform: ['twitter', 'linkedin', 'facebook', 'copy_link'], + }, + }, + { + name: 'scroll_depth', + weight: 0.2, + pages: blogPosts.map(slug => `/blog/${slug}`), + data: { + depth: [25, 50, 75, 100], + }, + }, +]; + +export function getBlogSiteConfig(): SiteConfig { + return { + hostname: BLOG_WEBSITE_DOMAIN, + pages: blogPages, + journeys: blogJourneys, + customEvents: blogCustomEvents, + }; +} + +export function getBlogJourney(): string[] { + const journeyWeights: WeightedOption<string[]>[] = blogJourneys.map(j => ({ + value: j.pages, + weight: j.weight, + })); + + return weightedRandom(journeyWeights); +} + +export const BLOG_SESSIONS_PER_DAY = 3; // ~90 sessions per month diff --git a/scripts/seed/sites/saas.ts b/scripts/seed/sites/saas.ts new file mode 100644 index 0000000..133895a --- /dev/null +++ b/scripts/seed/sites/saas.ts @@ -0,0 +1,185 @@ +import { weightedRandom, type WeightedOption } from '../utils.js'; +import type { + SiteConfig, + JourneyConfig, + PageConfig, + CustomEventConfig, +} from '../generators/events.js'; +import type { RevenueConfig } from '../generators/revenue.js'; + +export const SAAS_WEBSITE_NAME = 'Demo SaaS'; +export const SAAS_WEBSITE_DOMAIN = 'app.example.com'; + +const docsSections = [ + 'getting-started', + 'installation', + 'configuration', + 'api-reference', + 'integrations', +]; + +const blogPosts = [ + 'announcing-v2', + 'customer-success-story', + 'product-roadmap', + 'security-best-practices', +]; + +export const saasPages: PageConfig[] = [ + { path: '/', title: 'Demo SaaS - Analytics Made Simple', weight: 0.2, avgTimeOnPage: 45 }, + { path: '/features', title: 'Features', weight: 0.15, avgTimeOnPage: 90 }, + { path: '/pricing', title: 'Pricing', weight: 0.15, avgTimeOnPage: 120 }, + { path: '/docs', title: 'Documentation', weight: 0.1, avgTimeOnPage: 60 }, + { path: '/blog', title: 'Blog', weight: 0.05, avgTimeOnPage: 45 }, + { path: '/signup', title: 'Sign Up', weight: 0.08, avgTimeOnPage: 90 }, + { path: '/login', title: 'Login', weight: 0.05, avgTimeOnPage: 30 }, + { path: '/demo', title: 'Request Demo', weight: 0.05, avgTimeOnPage: 60 }, + ...docsSections.map(slug => ({ + path: `/docs/${slug}`, + title: `Docs: ${slug + .split('-') + .map(w => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' ')}`, + weight: 0.02, + avgTimeOnPage: 180, + })), + ...blogPosts.map(slug => ({ + path: `/blog/${slug}`, + title: slug + .split('-') + .map(w => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' '), + weight: 0.02, + avgTimeOnPage: 150, + })), +]; + +export const saasJourneys: JourneyConfig[] = [ + // Conversion funnel + { pages: ['/', '/features', '/pricing', '/signup'], weight: 0.12 }, + { pages: ['/', '/pricing', '/signup'], weight: 0.1 }, + { pages: ['/pricing', '/signup'], weight: 0.08 }, + + // Feature exploration + { pages: ['/', '/features'], weight: 0.1 }, + { pages: ['/', '/features', '/pricing'], weight: 0.08 }, + + // Documentation users + { pages: ['/docs', '/docs/getting-started'], weight: 0.08 }, + { pages: ['/docs/getting-started', '/docs/installation', '/docs/configuration'], weight: 0.06 }, + { pages: ['/docs/api-reference'], weight: 0.05 }, + + // Blog readers + { pages: ['/blog/announcing-v2'], weight: 0.05 }, + { pages: ['/blog/customer-success-story'], weight: 0.04 }, + + // Returning users + { pages: ['/login'], weight: 0.08 }, + + // Bounces + { pages: ['/'], weight: 0.08 }, + { pages: ['/pricing'], weight: 0.05 }, + + // Demo requests + { pages: ['/', '/demo'], weight: 0.03 }, +]; + +export const saasCustomEvents: CustomEventConfig[] = [ + { + name: 'signup_started', + weight: 0.6, + pages: ['/signup'], + data: { + plan: ['free', 'pro', 'enterprise'], + }, + }, + { + name: 'signup_completed', + weight: 0.3, + pages: ['/signup'], + data: { + plan: ['free', 'pro', 'enterprise'], + method: ['email', 'google', 'github'], + }, + }, + { + name: 'purchase', + weight: 0.15, + pages: ['/signup', '/pricing'], + data: { + plan: ['pro', 'enterprise'], + billing: ['monthly', 'annual'], + revenue: [29, 49, 99, 299], + currency: ['USD'], + }, + }, + { + name: 'demo_requested', + weight: 0.5, + pages: ['/demo'], + data: { + company_size: ['1-10', '11-50', '51-200', '200+'], + }, + }, + { + name: 'feature_viewed', + weight: 0.3, + pages: ['/features'], + data: { + feature: ['analytics', 'reports', 'api', 'integrations', 'privacy'], + }, + }, + { + name: 'cta_click', + weight: 0.15, + pages: ['/', '/features', '/pricing'], + data: { + button: ['hero_signup', 'nav_signup', 'pricing_cta', 'footer_cta'], + }, + }, + { + name: 'docs_search', + weight: 0.2, + pages: ['/docs', ...docsSections.map(s => `/docs/${s}`)], + data: { + query_type: ['api', 'setup', 'integration', 'troubleshooting'], + }, + }, +]; + +export const saasRevenueConfigs: RevenueConfig[] = [ + { + eventName: 'purchase', + minAmount: 29, + maxAmount: 29, + currency: 'USD', + weight: 0.7, // 70% Pro plan + }, + { + eventName: 'purchase', + minAmount: 299, + maxAmount: 299, + currency: 'USD', + weight: 0.3, // 30% Enterprise + }, +]; + +export function getSaasSiteConfig(): SiteConfig { + return { + hostname: SAAS_WEBSITE_DOMAIN, + pages: saasPages, + journeys: saasJourneys, + customEvents: saasCustomEvents, + }; +} + +export function getSaasJourney(): string[] { + const journeyWeights: WeightedOption<string[]>[] = saasJourneys.map(j => ({ + value: j.pages, + weight: j.weight, + })); + + return weightedRandom(journeyWeights); +} + +export const SAAS_SESSIONS_PER_DAY = 500; diff --git a/scripts/seed/utils.ts b/scripts/seed/utils.ts new file mode 100644 index 0000000..7b44261 --- /dev/null +++ b/scripts/seed/utils.ts @@ -0,0 +1,85 @@ +import { v4 as uuidv4 } from 'uuid'; + +export interface WeightedOption<T> { + value: T; + weight: number; +} + +export function weightedRandom<T>(options: WeightedOption<T>[]): T { + const totalWeight = options.reduce((sum, opt) => sum + opt.weight, 0); + let random = Math.random() * totalWeight; + + for (const option of options) { + random -= option.weight; + if (random <= 0) { + return option.value; + } + } + + return options[options.length - 1].value; +} + +export function randomInt(min: number, max: number): number { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +export function randomFloat(min: number, max: number): number { + return Math.random() * (max - min) + min; +} + +export function pickRandom<T>(array: T[]): T { + return array[Math.floor(Math.random() * array.length)]; +} + +export function shuffleArray<T>(array: T[]): T[] { + const result = [...array]; + for (let i = result.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [result[i], result[j]] = [result[j], result[i]]; + } + return result; +} + +export function uuid(): string { + return uuidv4(); +} + +export function generateDatesBetween(startDate: Date, endDate: Date): Date[] { + const dates: Date[] = []; + const current = new Date(startDate); + current.setHours(0, 0, 0, 0); + + while (current <= endDate) { + dates.push(new Date(current)); + current.setDate(current.getDate() + 1); + } + + return dates; +} + +export function addHours(date: Date, hours: number): Date { + return new Date(date.getTime() + hours * 60 * 60 * 1000); +} + +export function addMinutes(date: Date, minutes: number): Date { + return new Date(date.getTime() + minutes * 60 * 1000); +} + +export function addSeconds(date: Date, seconds: number): Date { + return new Date(date.getTime() + seconds * 1000); +} + +export function subDays(date: Date, days: number): Date { + return new Date(date.getTime() - days * 24 * 60 * 60 * 1000); +} + +export function formatNumber(num: number): string { + return num.toLocaleString(); +} + +export function progressBar(current: number, total: number, width = 30): string { + const percent = current / total; + const filled = Math.round(width * percent); + const empty = width - filled; + return `[${'█'.repeat(filled)}${'░'.repeat(empty)}] ${Math.round(percent * 100)}%`; +} diff --git a/scripts/start-env.js b/scripts/start-env.js new file mode 100644 index 0000000..970414a --- /dev/null +++ b/scripts/start-env.js @@ -0,0 +1,7 @@ +import 'dotenv/config'; +import cli from 'next/dist/cli/next-start'; + +cli.nextStart({ + port: process.env.PORT || 3000, + hostname: process.env.HOSTNAME || '0.0.0.0', +}); diff --git a/scripts/telemetry.js b/scripts/telemetry.js new file mode 100644 index 0000000..78bf54f --- /dev/null +++ b/scripts/telemetry.js @@ -0,0 +1,40 @@ +import os from 'node:os'; +import path from 'node:path'; +import isCI from 'is-ci'; +import { createRequire } from 'module'; + +const require = createRequire(import.meta.url); +const pkg = require(path.resolve(process.cwd(), 'package.json')); + +const url = 'https://api.umami.is/v1/telemetry'; + +export async function sendTelemetry(type) { + const { default: isDocker } = await import('is-docker'); + const { default: fetch } = await import('node-fetch'); + + const data = { + type, + payload: { + version: pkg.version, + node: process.version, + platform: os.platform(), + arch: os.arch(), + os: `${os.type()} ${os.version()}`, + is_docker: isDocker(), + is_ci: isCI, + }, + }; + + try { + await fetch(url, { + method: 'post', + cache: 'no-cache', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + } catch { + // Ignore + } +} diff --git a/scripts/update-tracker.js b/scripts/update-tracker.js new file mode 100644 index 0000000..0091e8b --- /dev/null +++ b/scripts/update-tracker.js @@ -0,0 +1,16 @@ +/* eslint-disable no-console */ +import 'dotenv/config'; +import fs from 'node:fs'; +import path from 'node:path'; + +const endPoint = process.env.COLLECT_API_ENDPOINT; + +if (endPoint) { + const file = path.resolve(process.cwd(), 'public/script.js'); + + const tracker = fs.readFileSync(file); + + fs.writeFileSync(path.resolve(file), tracker.toString().replace(/\/api\/send/g, endPoint)); + + console.log(`Updated tracker endpoint: ${endPoint}.`); +} |