package ingest import ( "context" "encoding/json" "fmt" "github.com/Fuwn/plutia/internal/config" "github.com/Fuwn/plutia/internal/storage" "github.com/Fuwn/plutia/internal/types" "math/rand" "os" "path/filepath" "testing" "time" ) func TestThinDiskFootprintFor1000DIDs(t *testing.T) { const didCount = 1000 dids := make([]string, 0, didCount) recordsByDID := make(map[string][]types.ExportRecord, didCount) mirrorRecords := make([]types.ExportRecord, 0, didCount) for i := 0; i < didCount; i++ { did := fmt.Sprintf("did:plc:thin-%04d", i) dids = append(dids, did) logRecords := signedDIDLog(t, did, 1) recordsByDID[did] = logRecords mirrorRecords = append(mirrorRecords, types.ExportRecord{ Seq: uint64(i + 1), DID: did, CreatedAt: logRecords[0].CreatedAt, CID: logRecords[0].CID, Operation: logRecords[0].Operation, }) } rng := rand.New(rand.NewSource(42)) rng.Shuffle(len(dids), func(i, j int) { dids[i], dids[j] = dids[j], dids[i] }) upstream := newThinUpstream(t, recordsByDID) defer upstream.close() thinDir := filepath.Join(t.TempDir(), "thin-data") if err := os.MkdirAll(thinDir, 0o755); err != nil { t.Fatalf("mkdir thin dir: %v", err) } thinStore, err := storage.OpenPebble(thinDir) if err != nil { t.Fatalf("open thin pebble: %v", err) } defer thinStore.Close() if err := thinStore.SetMode(config.ModeThin); err != nil { t.Fatalf("set thin mode: %v", err) } thinCfg := config.Default() thinCfg.Mode = config.ModeThin thinCfg.DataDir = thinDir thinCfg.PLCSource = upstream.url thinCfg.VerifyPolicy = config.VerifyFull thinCfg.ThinCacheTTL = 24 * time.Hour thinCfg.ThinCacheMaxEntries = didCount + 10 thinSvc := NewService(thinCfg, thinStore, NewClient(upstream.url), nil, nil) defer thinSvc.Close() for _, did := range dids { if _, err := thinSvc.ResolveState(context.Background(), did); err != nil { t.Fatalf("thin resolve %s: %v", did, err) } } thinSize, err := dirSizeForTest(thinDir) if err != nil { t.Fatalf("thin dir size: %v", err) } mirrorDir := filepath.Join(t.TempDir(), "mirror-data") if err := os.MkdirAll(mirrorDir, 0o755); err != nil { t.Fatalf("mkdir mirror dir: %v", err) } mirrorStore, err := storage.OpenPebble(mirrorDir) if err != nil { t.Fatalf("open mirror pebble: %v", err) } defer mirrorStore.Close() if err := mirrorStore.SetMode(config.ModeMirror); err != nil { t.Fatalf("set mirror mode: %v", err) } mirrorLog, err := storage.OpenBlockLog(mirrorDir, 9, 4) if err != nil { t.Fatalf("open mirror blocklog: %v", err) } exportPath := filepath.Join(mirrorDir, "mirror-source.ndjson") f, err := os.Create(exportPath) if err != nil { t.Fatalf("create mirror source: %v", err) } enc := json.NewEncoder(f) for _, rec := range mirrorRecords { if err := enc.Encode(rec); err != nil { _ = f.Close() t.Fatalf("encode mirror source: %v", err) } } if err := f.Close(); err != nil { t.Fatalf("close mirror source: %v", err) } mirrorCfg := config.Default() mirrorCfg.Mode = config.ModeMirror mirrorCfg.DataDir = mirrorDir mirrorCfg.PLCSource = exportPath mirrorCfg.VerifyPolicy = config.VerifyFull mirrorCfg.ZstdLevel = 9 mirrorCfg.BlockSizeMB = 4 mirrorCfg.CommitBatchSize = 128 mirrorCfg.VerifyWorkers = 8 mirrorCfg.CheckpointInterval = 1000000 mirrorSvc := NewService(mirrorCfg, mirrorStore, NewClient(exportPath), mirrorLog, nil) defer mirrorSvc.Close() if err := mirrorSvc.Replay(context.Background()); err != nil { t.Fatalf("mirror replay: %v", err) } if err := mirrorSvc.Flush(context.Background()); err != nil { t.Fatalf("mirror flush: %v", err) } mirrorSize, err := dirSizeForTest(mirrorDir) if err != nil { t.Fatalf("mirror dir size: %v", err) } if thinSize >= mirrorSize { t.Fatalf("expected thin footprint smaller than mirror footprint: thin=%d mirror=%d", thinSize, mirrorSize) } t.Logf("thin_cache_total_bytes=%d did_count=%d bytes_per_did=%.2f", thinSize, didCount, float64(thinSize)/didCount) t.Logf("mirror_total_bytes=%d op_count=%d bytes_per_op=%.2f", mirrorSize, didCount, float64(mirrorSize)/didCount) } func dirSizeForTest(path string) (int64, error) { var total int64 err := filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } info, err := d.Info() if err != nil { return err } total += info.Size() return nil }) if err != nil { return 0, err } return total, nil }