diff --git a/apps/frontend/app/api/daily-players/route.ts b/apps/frontend/app/api/daily-players/route.ts index 77707db..95f53c4 100644 --- a/apps/frontend/app/api/daily-players/route.ts +++ b/apps/frontend/app/api/daily-players/route.ts @@ -3,7 +3,10 @@ import { CACHE_SECONDS, getDailyPlayers } from '../../../utils/dailyPlayers'; export async function GET(request: NextRequest) { return NextResponse.json( - await getDailyPlayers(request.nextUrl.searchParams.get('range') ?? '90d'), + await getDailyPlayers( + request.nextUrl.searchParams.get('range') ?? '90d', + request.nextUrl.searchParams.get('source') ?? 'teerank' + ), { headers: { 'Cache-Control': `public, max-age=${CACHE_SECONDS}, stale-while-revalidate=86400`, diff --git a/apps/frontend/app/gametype/[gameTypeName]/(lists)/layout.tsx b/apps/frontend/app/gametype/[gameTypeName]/(lists)/layout.tsx index 23744e4..69bbb71 100644 --- a/apps/frontend/app/gametype/[gameTypeName]/(lists)/layout.tsx +++ b/apps/frontend/app/gametype/[gameTypeName]/(lists)/layout.tsx @@ -44,6 +44,7 @@ export default async function Index({ playerCount={gameType.playerCount} clanCount={gameType.clanCount} serverCount={gameType.gameServerCount} + showRecords={gameTypeName === 'DDraceNetwork'} /> {children} diff --git a/apps/frontend/app/gametype/[gameTypeName]/(lists)/records/page.tsx b/apps/frontend/app/gametype/[gameTypeName]/(lists)/records/page.tsx new file mode 100644 index 0000000..a480ae3 --- /dev/null +++ b/apps/frontend/app/gametype/[gameTypeName]/(lists)/records/page.tsx @@ -0,0 +1,57 @@ +import { Metadata } from 'next'; +import { notFound } from 'next/navigation'; +import { z } from 'zod'; +import { format } from 'date-fns'; +import { paramsSchema } from '../../schema'; +import { encodeString } from '../../../../../utils/encoding'; +import { listRecentRecords } from '../../../../../utils/ddnetMap'; +import { RecordList } from '../../../../../components/RecordList'; +import { DDNetAttribution } from '../../../../../components/DDNetAttribution'; + +export async function generateMetadata({ + params, +}: { + params: z.infer; +}): Promise { + const { gameTypeName } = paramsSchema.parse(params); + + return { + title: `Records - ${gameTypeName}`, + description: `Latest map records in ${gameTypeName}`, + alternates: { + canonical: `https://teerank.io/gametype/${encodeString(gameTypeName)}/records`, + }, + }; +} + +export default async function Index({ + params, +}: { + params: { [key: string]: string }; +}) { + const { gameTypeName } = paramsSchema.parse(params); + + if (gameTypeName !== 'DDraceNetwork') { + return notFound(); + } + + const records = await listRecentRecords(); + + return ( + <> + ({ + map: record.map, + players: record.players, + time: record.time, + delta: record.delta, + dateLabel: format(record.date, 'MMM d, yyyy'), + }))} + showMap + showDelta + /> + + + + ); +} diff --git a/apps/frontend/app/gametype/[gameTypeName]/LayoutTabs.tsx b/apps/frontend/app/gametype/[gameTypeName]/LayoutTabs.tsx index b62fbe8..f83a79d 100644 --- a/apps/frontend/app/gametype/[gameTypeName]/LayoutTabs.tsx +++ b/apps/frontend/app/gametype/[gameTypeName]/LayoutTabs.tsx @@ -10,12 +10,14 @@ export function LayoutTabs({ playerCount, clanCount, serverCount, + showRecords, }: { gameTypeName: string; mapName?: string; playerCount: number; clanCount: number; serverCount: number; + showRecords?: boolean; }) { const pathname = usePathname(); @@ -45,6 +47,13 @@ export function LayoutTabs({ isActive={pathname === `${urlPathname}/servers`} href={{ pathname: `${urlPathname}/servers` }} /> + {showRecords && ( + + )} ); } diff --git a/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/layout.tsx b/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/layout.tsx index 46e7770..3f4ae10 100644 --- a/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/layout.tsx +++ b/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/layout.tsx @@ -1,11 +1,76 @@ import { notFound } from 'next/navigation'; import Link from 'next/link'; +import { Fragment } from 'react'; +import { format } from 'date-fns'; +import { localizeUtcDay } from '@teerank/teerank/date'; import prisma from '../../../../../utils/prisma'; import { LayoutTabs } from '../../LayoutTabs'; import { paramsSchema } from './schema'; import { encodeString } from '../../../../../utils/encoding'; import { ActivityHeader } from '../../../../../components/ActivityHeader'; import { getMapActivity } from '../../../../../utils/activity'; +import { getDdnetMapHeader } from '../../../../../utils/ddnetMap'; +import { formatInteger } from '../../../../../utils/format'; + +function MapperLinks({ ddnetMap, mappers }: { ddnetMap: { mapper: string }; mappers: string[] }) { + if (mappers.length === 0) { + return <>{ddnetMap.mapper}; + } + + return ( + <> + {mappers.map((mapper, index) => ( + + {index > 0 && ', '} + + {mapper} + + + ))} + + ); +} + +function DdnetMapStats({ + ddnetMap, + recordsPath, +}: { + ddnetMap: { + category: string; + points: number; + releasedAt: Date | null; + finishCount: number; + tiles: string[]; + }; + recordsPath: string; +}) { + const tiles = ddnetMap.tiles.map((tile) => tile.toLowerCase().replace(/_/g, ' ')); + + return ( + <> + 0 ? 'cursor-help' : ''}`} + title={tiles.length > 0 ? tiles.join(', ') : undefined} + > + {ddnetMap.category} · {ddnetMap.points} pts + + {ddnetMap.releasedAt !== null && ( + + Released {format(localizeUtcDay(ddnetMap.releasedAt), 'MMM yyyy')} + + )} + + + {formatInteger(ddnetMap.finishCount)} finishes + + + + ); +} export default async function Index({ children, @@ -29,7 +94,10 @@ export default async function Index({ notFound(); } - const activity = await getMapActivity(map.id, { range: '1y' }); + const [activity, ddnet] = await Promise.all([ + getMapActivity(map.id, { range: '1y' }), + gameTypeName === 'DDraceNetwork' ? getDdnetMapHeader(map.id) : null, + ]); return (
@@ -39,15 +107,48 @@ export default async function Index({ activity={activity} contentClassName="flex-col justify-center gap-2" > -

{mapName}

- - - {gameTypeName} - - +
+ {ddnet !== null && ddnet.hasThumb && ( + {`${mapName} + )} +
+

+ {mapName} + {ddnet !== null && ddnet.ddnetMap.stars > 0 && ( + + {'★'.repeat(ddnet.ddnetMap.stars)} + + )} + {ddnet !== null && ( + + by + + )} +

+
+ + + {gameTypeName} + + + {ddnet !== null && ( + + )} +
+
+
@@ -57,6 +158,7 @@ export default async function Index({ playerCount={map.playerCount} clanCount={map.clanCount} serverCount={map.gameServerCount} + showRecords={ddnet !== null} /> {children} diff --git a/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/records/page.tsx b/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/records/page.tsx new file mode 100644 index 0000000..da2ef35 --- /dev/null +++ b/apps/frontend/app/gametype/[gameTypeName]/map/[mapName]/records/page.tsx @@ -0,0 +1,247 @@ +import { Metadata } from 'next'; +import { Fragment } from 'react'; +import { notFound } from 'next/navigation'; +import { z } from 'zod'; +import { format } from 'date-fns'; +import { localizeUtcDay } from '@teerank/teerank/date'; +import prisma from '../../../../../../utils/prisma'; +import { paramsSchema } from '../schema'; +import { searchParamPageSchema } from '../../../../../../utils/page'; +import { sharedHiddenParam } from '../../../../../../utils/shared'; +import { encodeString } from '../../../../../../utils/encoding'; +import { formatFinishTime, formatTimeDelta } from '../../../../../../utils/format'; +import { + getCompareSplits, + getDdnetMap, + getRecordEvents, + listSoloRecords, + listTeamRecords, +} from '../../../../../../utils/ddnetMap'; +import { StepLineChart } from '../../../../../../components/Chart'; +import { RecordList, RecordRanks } from '../../../../../../components/RecordList'; +import { DDNetAttribution } from '../../../../../../components/DDNetAttribution'; + +export async function generateMetadata({ + params, +}: { + params: z.infer; +}): Promise { + const { gameTypeName, mapName } = paramsSchema.parse(params); + + return { + title: `${mapName} records - ${gameTypeName}`, + description: `Record times for ${mapName} in ${gameTypeName}`, + alternates: { + canonical: `https://teerank.io/gametype/${encodeString(gameTypeName)}/map/${encodeString(mapName)}/records`, + }, + }; +} + +function Checkpoints({ + wrSplits, + medianSplits, + compareName, + compareSplits, +}: { + wrSplits: number[]; + medianSplits: number[]; + compareName?: string; + compareSplits: number[] | null; +}) { + const showCompare = compareName !== undefined && compareSplits !== null; + + const rows = wrSplits.flatMap((wrSplit, index) => + wrSplit > 0 + ? [ + { + checkpoint: index + 1, + wrSplit, + medianSplit: medianSplits[index] ?? 0, + compareSplit: compareSplits?.[index] ?? 0, + }, + ] + : [] + ); + + const losses = rows.map((row) => + row.medianSplit > 0 ? row.medianSplit - row.wrSplit : null + ); + const maxLoss = Math.max( + ...losses.filter((loss): loss is number => loss !== null), + 0 + ); + + const columnCount = showCompare ? 6 : 4; + + return ( +
+ + Checkpoints + +
+ CP + Record split + Median split + Time lost + {showCompare && ( + + {compareName} + + )} + {showCompare && ( + Δ vs record + )} + + + + {rows.map((row, index) => ( + + {row.checkpoint} + {formatFinishTime(row.wrSplit)} + + {row.medianSplit > 0 ? formatFinishTime(row.medianSplit) : ''} + + + {losses[index] === null ? '' : formatTimeDelta(losses[index] as number)} + + {showCompare && ( + + {row.compareSplit > 0 ? formatFinishTime(row.compareSplit) : ''} + + )} + {showCompare && ( + + {row.compareSplit > 0 + ? formatTimeDelta(row.compareSplit - row.wrSplit) + : ''} + + )} + + ))} +
+
+ ); +} + +export default async function Index({ + params, + searchParams, +}: { + params: { [key: string]: string }; + searchParams: { [key: string]: string | string[] | undefined }; +}) { + const { gameTypeName, mapName } = paramsSchema.parse(params); + const { page } = searchParamPageSchema.parse(searchParams); + + const ranks: RecordRanks = + searchParams['ranks'] === 'team' + ? 'team' + : searchParams['ranks'] === 'duo' + ? 'duo' + : 'solo'; + const compareName = + typeof searchParams['compare'] === 'string' + ? searchParams['compare'] + : undefined; + + const map = await prisma.map.findUnique({ + select: { + id: true, + }, + where: { + name_gameTypeName: { + name: mapName, + gameTypeName: gameTypeName, + }, + }, + }); + + if (map === null) { + return notFound(); + } + + const ddnetMap = await getDdnetMap(map.id); + + if (ddnetMap === null) { + return notFound(); + } + + const hideShared = ranks === 'solo' && sharedHiddenParam(searchParams); + + const [events, list, compareSplits] = await Promise.all([ + getRecordEvents(map.id), + ranks === 'solo' + ? listSoloRecords({ mapId: map.id, page, hideShared }) + : listTeamRecords({ + mapId: map.id, + page, + size: ranks === 'duo' ? 2 : undefined, + }), + compareName === undefined || ddnetMap.wrSplits.length === 0 + ? null + : getCompareSplits(map.id, compareName), + ]); + + const pathname = `/gametype/${encodeString(gameTypeName)}/map/${encodeString(mapName)}/records`; + + return ( + <> + {events.length >= 2 && ( +
+ ({ + date: event.date, + value: event.time, + label: `${formatFinishTime(event.time)} by ${event.names.join(', ')} — ${format(event.date, 'MMM yyyy')}`, + }))} + endDate={new Date()} + /> +
+ )} + + ({ + rank: record.rank, + players: record.players, + time: record.time, + dateLabel: + ranks === 'solo' + ? format(localizeUtcDay(record.date), 'MMM d, yyyy') + : format(record.date, 'MMM d, yyyy'), + }))} + recordCount={list.count} + ranks={ranks} + pathname={pathname} + showRank + showSharedToggle={ranks === 'solo'} + /> + + {ddnetMap.wrSplits.length > 0 && ( + + )} + + + + ); +} diff --git a/apps/frontend/app/map-thumb/[mapId]/route.ts b/apps/frontend/app/map-thumb/[mapId]/route.ts new file mode 100644 index 0000000..ec9b2d1 --- /dev/null +++ b/apps/frontend/app/map-thumb/[mapId]/route.ts @@ -0,0 +1,29 @@ +import { notFound } from "next/navigation"; +import prisma from "../../../utils/prisma"; + +export async function GET( + _request: Request, + { params }: { params: { mapId: string } } +) { + const mapId = Number(params.mapId); + + if (!Number.isInteger(mapId)) { + notFound(); + } + + const thumb = await prisma.ddnetMapThumb.findUnique({ + where: { mapId }, + select: { png: true }, + }); + + if (thumb === null) { + notFound(); + } + + return new Response(Buffer.from(thumb.png), { + headers: { + 'Content-Type': 'image/png', + 'Cache-Control': 'public, max-age=86400, stale-while-revalidate=604800', + }, + }); +} diff --git a/apps/frontend/app/mapper/[mapperName]/page.tsx b/apps/frontend/app/mapper/[mapperName]/page.tsx new file mode 100644 index 0000000..c8ec177 --- /dev/null +++ b/apps/frontend/app/mapper/[mapperName]/page.tsx @@ -0,0 +1,109 @@ +import { Metadata } from 'next'; +import { notFound } from 'next/navigation'; +import { Fragment } from 'react'; +import { z } from 'zod'; +import { format } from 'date-fns'; +import { localizeUtcDay } from '@teerank/teerank/date'; +import { paramsSchema } from './schema'; +import { encodeString } from '../../../utils/encoding'; +import { formatFinishTime, formatInteger } from '../../../utils/format'; +import { getMapperMaps } from '../../../utils/ddnetMap'; +import { List, ListCell } from '../../../components/List'; +import { DDNetAttribution } from '../../../components/DDNetAttribution'; + +export async function generateMetadata({ + params, +}: { + params: z.infer; +}): Promise { + const { mapperName } = paramsSchema.parse(params); + + return { + title: `Mapper ${mapperName}`, + description: `Maps made by ${mapperName}`, + alternates: { + canonical: `https://teerank.io/mapper/${encodeString(mapperName)}`, + }, + }; +} + +export default async function Index({ + params, +}: { + params: { [key: string]: string }; +}) { + const { mapperName } = paramsSchema.parse(params); + + const maps = await getMapperMaps(mapperName); + + if (maps === null) { + return notFound(); + } + + const totalFinishes = maps.reduce((sum, map) => sum + map.finishCount, 0); + const releaseYears = maps.flatMap((map) => + map.releasedAt === null ? [] : [localizeUtcDay(map.releasedAt).getFullYear()] + ); + + const minYear = Math.min(...releaseYears); + const maxYear = Math.max(...releaseYears); + const releasedLabel = + releaseYears.length === 0 + ? '' + : minYear === maxYear + ? ` · released ${minYear}` + : ` · released ${minYear}–${maxYear}`; + + return ( +
+
+

{mapperName}

+ + {formatInteger(maps.length)} {maps.length === 1 ? 'map' : 'maps'} ·{' '} + {formatInteger(totalFinishes)} total finishes + {releasedLabel} + + +
+ + + {maps.map((map) => ( + + + + + + + + + ))} + +
+ ); +} diff --git a/apps/frontend/app/mapper/[mapperName]/schema.ts b/apps/frontend/app/mapper/[mapperName]/schema.ts new file mode 100644 index 0000000..f7d8645 --- /dev/null +++ b/apps/frontend/app/mapper/[mapperName]/schema.ts @@ -0,0 +1,6 @@ +import { z } from "zod"; +import { decodeString } from "../../../utils/encoding"; + +export const paramsSchema = z.object({ + mapperName: z.string().transform(decodeString), +}); diff --git a/apps/frontend/app/page.tsx b/apps/frontend/app/page.tsx index 7597343..cc11830 100644 --- a/apps/frontend/app/page.tsx +++ b/apps/frontend/app/page.tsx @@ -172,6 +172,17 @@ export default async function Index() {
+
+
+

DDNet integration

+

August 28th 2026

+
+

+ Teerank now imports data from DDNet daily and shows for each map + all finishes: solo, team and duo. It also gathers map metadata. +

+
+

Teammates, activity and shared player names

diff --git a/apps/frontend/app/player/[playerName]/LayoutTabs.tsx b/apps/frontend/app/player/[playerName]/LayoutTabs.tsx index f52195d..92403b4 100644 --- a/apps/frontend/app/player/[playerName]/LayoutTabs.tsx +++ b/apps/frontend/app/player/[playerName]/LayoutTabs.tsx @@ -8,10 +8,12 @@ export function LayoutTabs({ playerName, clanCount, teammateCount, + ddnetFinishCount, }: { playerName: string; clanCount: number; teammateCount: number; + ddnetFinishCount?: number; }) { const urlPathname = `/player/${encodeString(playerName)}`; const pathname = usePathname(); @@ -40,6 +42,14 @@ export function LayoutTabs({ isActive={pathname === `${urlPathname}/teammates`} href={{ pathname: `${urlPathname}/teammates` }} /> + {ddnetFinishCount !== undefined && ( + + )} ); } diff --git a/apps/frontend/app/player/[playerName]/ddnet/page.tsx b/apps/frontend/app/player/[playerName]/ddnet/page.tsx new file mode 100644 index 0000000..f33bb75 --- /dev/null +++ b/apps/frontend/app/player/[playerName]/ddnet/page.tsx @@ -0,0 +1,198 @@ +import { paramsSchema } from '../schema'; +import { z } from 'zod'; +import { notFound } from 'next/navigation'; +import Link from 'next/link'; +import { Metadata } from 'next'; +import { Fragment } from 'react'; +import prisma from '../../../../utils/prisma'; +import { searchParamPageSchema } from '../../../../utils/page'; +import { List, ListCell } from '../../../../components/List'; +import { DDNetAttribution } from '../../../../components/DDNetAttribution'; +import { encodeString } from '../../../../utils/encoding'; +import { formatFinishTime, formatInteger } from '../../../../utils/format'; +import { + countDdnetBestTimes, + formatUtcDate, + getDdnetBestTimes, + getDdnetJourney, + getDdnetPlayer, +} from '../../../../utils/ddnetPlayer'; + +export async function generateMetadata({ + params, +}: { + params: z.infer; +}): Promise { + const { playerName } = paramsSchema.parse(params); + + return { + title: `Player ${playerName} - DDNet`, + description: 'DDNet finishes and best times of a Teeworlds player', + alternates: { + canonical: `https://teerank.io/player/${encodeString(playerName)}/ddnet`, + }, + }; +} + +function mapHref(mapName: string) { + return { + pathname: `/gametype/DDraceNetwork/map/${encodeString(mapName)}`, + }; +} + +export default async function Index({ + params, + searchParams, +}: { + params: z.infer; + searchParams: { [key: string]: string | string[] | undefined }; +}) { + const { playerName } = paramsSchema.parse(params); + const { page } = searchParamPageSchema.parse(searchParams); + + const player = await prisma.player.findUnique({ + select: { + id: true, + }, + where: { + name: playerName, + }, + }); + + if (player === null) { + return notFound(); + } + + const ddnetPlayer = await getDdnetPlayer(player.id); + + if (ddnetPlayer === null) { + return notFound(); + } + + const [journey, bestTimes, bestTimeCount] = await Promise.all([ + getDdnetJourney(player.id, ddnetPlayer.lastFinishAt), + getDdnetBestTimes(player.id, { skip: (page - 1) * 100, take: 100 }), + countDdnetBestTimes(player.id), + ]); + + return ( + <> +
+

Journey

+ + {journey.tiers.map((tier) => ( + + + + + + + + ))} + +

+ {'First finish: '} + {journey.firstMapName !== null && ( + <> + + {journey.firstMapName} + + {' · '} + + )} + {formatUtcDate(ddnetPlayer.firstFinishAt, 'MMM yyyy')} + {' — Last finish: '} + {journey.lastMapName !== null && ( + <> + + {journey.lastMapName} + + {' · '} + + )} + {formatUtcDate(ddnetPlayer.lastFinishAt, 'MMM yyyy')} +

+
+ +
+

Best times

+ + {bestTimes.map((bestTime) => ( + + + + + + + + + + ))} + + +
+ + ); +} diff --git a/apps/frontend/app/player/[playerName]/layout.tsx b/apps/frontend/app/player/[playerName]/layout.tsx index c0ae3e1..2c47b40 100644 --- a/apps/frontend/app/player/[playerName]/layout.tsx +++ b/apps/frontend/app/player/[playerName]/layout.tsx @@ -6,7 +6,7 @@ import Image from 'next/image'; import prisma from '../../../utils/prisma'; import { LayoutTabs } from './LayoutTabs'; import { LastSeen } from '../../../components/LastSeen'; -import { formatPlayTime } from '../../../utils/format'; +import { formatInteger, formatPlayTime } from '../../../utils/format'; import { encodeString } from '../../../utils/encoding'; import { ActivityHeader } from '../../../components/ActivityHeader'; import { SharedRatio } from '../../../components/SharedRatio'; @@ -56,7 +56,7 @@ export default async function Index({ return notFound(); } - const [clanCount, teammateCount, activity] = await Promise.all([ + const [clanCount, teammateCount, activity, ddnetPlayer] = await Promise.all([ prisma.clanPlayerInfo.count({ where: { playerName, @@ -64,6 +64,16 @@ export default async function Index({ }), countTeammates(player.id), getPlayerActivity(player.id, { range: '1y' }), + prisma.ddnetPlayer.findUnique({ + select: { + points: true, + pointsRank: true, + finishCount: true, + }, + where: { + playerId: player.id, + }, + }), ]); return ( @@ -108,6 +118,13 @@ export default async function Index({ playerName={player.name} /> + {ddnetPlayer !== null && ( + + DDNet: {formatInteger(ddnetPlayer.points)} pts + {ddnetPlayer.pointsRank !== null && + ` · #${formatInteger(ddnetPlayer.pointsRank)}`} + + )}
@@ -117,6 +134,7 @@ export default async function Index({ playerName={playerName} clanCount={clanCount} teammateCount={teammateCount} + ddnetFinishCount={ddnetPlayer?.finishCount} /> {children} diff --git a/apps/frontend/app/player/[playerName]/teammates/page.tsx b/apps/frontend/app/player/[playerName]/teammates/page.tsx index 4e5193a..7f5f36d 100644 --- a/apps/frontend/app/player/[playerName]/teammates/page.tsx +++ b/apps/frontend/app/player/[playerName]/teammates/page.tsx @@ -2,11 +2,30 @@ import { paramsSchema } from '../schema'; import { z } from 'zod'; import { notFound } from 'next/navigation'; import { Metadata } from 'next'; +import Link from 'next/link'; +import { Fragment } from 'react'; +import { intervalToDuration } from 'date-fns'; import prisma from '../../../../utils/prisma'; import { searchParamPageSchema } from '../../../../utils/page'; -import { TeammateList } from '../../../../components/TeammateList'; +import { List, ListCell } from '../../../../components/List'; +import { durationClassName, LastSeen } from '../../../../components/LastSeen'; +import { SharedRatio } from '../../../../components/SharedRatio'; +import { DDNetAttribution } from '../../../../components/DDNetAttribution'; import { encodeString } from '../../../../utils/encoding'; +import { + formatDurationShort, + formatFinishTime, + formatInteger, + formatPlayTime, +} from '../../../../utils/format'; import { countTeammates, getTeammates } from '../../../../utils/teammates'; +import { + countDdnetPartnerMaps, + countDdnetPartners, + formatUtcDate, + getDdnetPartnerMaps, + getDdnetPartners, +} from '../../../../utils/ddnetPlayer'; export async function generateMetadata({ params, @@ -24,6 +43,198 @@ export async function generateMetadata({ }; } +function mapHref(mapName: string) { + return { + pathname: `/gametype/DDraceNetwork/map/${encodeString(mapName)}`, + }; +} + +function SourceToggle({ + pathname, + label, + query, +}: { + pathname: string; + label: string; + query?: { [key: string]: string }; +}) { + return ( + + {label} + + ); +} + +async function PartnerMapList({ + playerId, + playerName, + partnerName, + page, +}: { + playerId: number; + playerName: string; + partnerName: string; + page: number; +}) { + const partner = await prisma.player.findUnique({ + select: { + id: true, + }, + where: { + name: partnerName, + }, + }); + + if (partner === null) { + return notFound(); + } + + const pathname = `/player/${encodeString(playerName)}/teammates`; + + const [rows, count] = await Promise.all([ + getDdnetPartnerMaps(playerId, partner.id, { + skip: (page - 1) * 100, + take: 100, + }), + countDdnetPartnerMaps(playerId, partner.id), + ]); + + return ( + <> +
+

+ with{' '} + + {partnerName} + +

+ + Back to partners + +
+ + {rows.map((row) => ( + + + + + + + + ))} + + + + ); +} + +async function PartnerList({ + playerId, + playerName, + page, +}: { + playerId: number; + playerName: string; + page: number; +}) { + const pathname = `/player/${encodeString(playerName)}/teammates`; + + const [partners, partnerCount] = await Promise.all([ + getDdnetPartners(playerId, { skip: (page - 1) * 100, take: 100 }), + countDdnetPartners(playerId), + ]); + + return ( + <> + + Name + + + ), + expand: true, + }, + { title: 'Finishes together', expand: false }, + { title: 'Best time', expand: true }, + { title: 'Last together', expand: false }, + ]} + pageCount={Math.ceil(partnerCount / 100)} + > + {partners.map((partner) => ( + + + + {partner.name} + + + + + + {formatFinishTime(partner.bestTime)} + {partner.bestMapName !== null && ( + <> + {' · '} + + {partner.bestMapName} + + + )} + + + + ))} + + + + ); +} + export default async function Index({ params, searchParams, @@ -33,6 +244,9 @@ export default async function Index({ }) { const { playerName } = paramsSchema.parse(params); const { page } = searchParamPageSchema.parse(searchParams); + const source = searchParams['source'] === 'finishes' ? 'finishes' : 'polled'; + const withName = + typeof searchParams['with'] === 'string' ? searchParams['with'] : undefined; const player = await prisma.player.findUnique({ select: { @@ -47,15 +261,97 @@ export default async function Index({ return notFound(); } - const [teammates, teammateCount] = await Promise.all([ + if (source === 'finishes') { + if (withName !== undefined) { + return ( + + ); + } + + return ( + + ); + } + + const pathname = `/player/${encodeString(playerName)}/teammates`; + + const [teammates, teammateCount, partnerCount] = await Promise.all([ getTeammates(player.id, { skip: (page - 1) * 100, take: 100 }), countTeammates(player.id), + countDdnetPartners(player.id), ]); return ( - + Name + {partnerCount > 0 && ( + + )} + + ), + expand: true, + }, + { title: 'Clan', expand: true }, + { title: 'Last seen', expand: false }, + { title: 'Coplaytime', expand: false }, + { title: 'Last together', expand: false }, + ]} pageCount={Math.ceil(teammateCount / 100)} - /> + > + {teammates.map((teammate) => { + const lastTogether = intervalToDuration({ + start: teammate.lastPlayedAt, + end: new Date(), + }); + + return ( + + + + + + + + ); + })} + ); } diff --git a/apps/frontend/app/status/page.tsx b/apps/frontend/app/status/page.tsx index bca7630..b3f2165 100644 --- a/apps/frontend/app/status/page.tsx +++ b/apps/frontend/app/status/page.tsx @@ -9,6 +9,9 @@ import { getArchiveSnapshotsFailedCount, getRollupDayFailedCount, getRollupBackfillFailedCount, + getDdnetBackfillFailedCount, + getDdnetImportFailedCount, + getDdnetOnlineFailedCount, SNAPSHOT_RETENTION_HOURS, DAY_MS, addUtcDays, @@ -47,6 +50,10 @@ export default async function Index() { rollupDayCount, rollupDayFailedCount, rollupBackfillFailedCount, + ddnetFailedCounts, + ddnetBackfillState, + ddnetImportState, + ddnetOnlineDayCount, oldestSnapshot, masterServers, unreferencedGameServersCount, @@ -66,6 +73,14 @@ export default async function Index() { prisma.globalDay.count(), getRollupDayFailedCount(), getRollupBackfillFailedCount(), + Promise.all([ + getDdnetBackfillFailedCount(), + getDdnetImportFailedCount(), + getDdnetOnlineFailedCount(), + ]).then((counts) => counts.reduce((sum, count) => sum + count, 0)), + prisma.ddnetState.findUnique({ where: { key: 'backfill' } }), + prisma.ddnetState.findUnique({ where: { key: 'import' } }), + prisma.ddnetOnlineDay.count({ where: { key: 'ALL' } }), prisma.gameServerSnapshot.findFirst({ orderBy: { id: 'asc', @@ -166,6 +181,18 @@ export default async function Index() { const rollupFailedCount = rollupDayFailedCount + rollupBackfillFailedCount; + const backfillPhase = + (ddnetBackfillState?.value as { phase?: string; monthIndex?: number; raceMonths?: string[] } | null) + ?? null; + const ddnetBackfillLabel = + backfillPhase === null + ? 'Not started' + : backfillPhase.phase === 'apply' && backfillPhase.raceMonths !== undefined + ? `apply ${backfillPhase.monthIndex ?? 0}/${backfillPhase.raceMonths.length} months` + : backfillPhase.phase ?? 'unknown'; + const ddnetLastDump = + (ddnetImportState?.value as { lastModified?: string } | null)?.lastModified ?? null; + return (

Teerank

@@ -303,6 +330,47 @@ export default async function Index() { +

DDNet import

+
+
+ Backfill +
+ {ddnetBackfillLabel} + {backfillPhase?.phase === 'done' && ( + Done + )} +
+
+ +
+ Last dump +
+ + {ddnetLastDump === null + ? 'None' + : formatDistanceToNow(new Date(ddnetLastDump), { addSuffix: true })} + +
+
+ +
+ Online history +
+ {ddnetOnlineDayCount} days +
+
+ +
+ Failed jobs +
+ {ddnetFailedCounts} + {ddnetFailedCounts > 0 && ( + Failing + )} +
+
+
+

Teeworlds

{masterServers.map((masterServer) => ( diff --git a/apps/frontend/components/ActivityCalendar.tsx b/apps/frontend/components/ActivityCalendar.tsx index bfa00ed..d7b901f 100644 --- a/apps/frontend/components/ActivityCalendar.tsx +++ b/apps/frontend/components/ActivityCalendar.tsx @@ -8,6 +8,7 @@ export type ActivityDay = { day: string; // YYYY-MM-DD value: number; tooltip: string; + kind?: 'finishes'; }; export type ActivityPayload = { @@ -19,6 +20,7 @@ export type ActivityPayload = { const EMPTY_COLOR = '#00000010'; const LEVEL_COLORS = ['#e6dcae', '#d2ba6a', '#b3922e', '#997700']; +const FINISH_LEVEL_COLORS = ['#d3dade', '#adbcc5', '#8299a7', '#5f7a8c']; const TOOLTIP_DELAY_MS = 300; @@ -26,18 +28,26 @@ function quantile(sorted: number[], q: number) { return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * q))]; } -function cellColor(value: number | undefined, thresholds: number[]) { - if (value === undefined || value <= 0) { +function thresholdsOf(values: number[]) { + const positives = values.filter((value) => value > 0).sort((a, b) => a - b); + return [quantile(positives, 0.25), quantile(positives, 0.5), quantile(positives, 0.75)]; +} + +function cellColor(day: ActivityDay | undefined, thresholds: number[], finishThresholds: number[]) { + if (day === undefined || day.value <= 0) { return EMPTY_COLOR; } - for (const [level, threshold] of thresholds.entries()) { - if (value <= threshold) { - return LEVEL_COLORS[level]; + const colors = day.kind === 'finishes' ? FINISH_LEVEL_COLORS : LEVEL_COLORS; + const levels = day.kind === 'finishes' ? finishThresholds : thresholds; + + for (const [level, threshold] of levels.entries()) { + if (day.value <= threshold) { + return colors[level]; } } - return LEVEL_COLORS[LEVEL_COLORS.length - 1]; + return colors[colors.length - 1]; } type Tooltip = { @@ -91,11 +101,12 @@ export function ActivityCalendar({ const to = parseUtcDay(payload.to); const days = new Map(payload.days.map((day) => [day.day, day])); - const positives = payload.days - .map(({ value }) => value) - .filter((value) => value > 0) - .sort((a, b) => a - b); - const thresholds = [quantile(positives, 0.25), quantile(positives, 0.5), quantile(positives, 0.75)]; + const thresholds = thresholdsOf( + payload.days.filter((day) => day.kind === undefined).map(({ value }) => value) + ); + const finishThresholds = thresholdsOf( + payload.days.filter((day) => day.kind === 'finishes').map(({ value }) => value) + ); // Columns start on the Sunday on or before the first day. const startSunday = addUtcDays(from, -from.getUTCDay()); @@ -135,7 +146,7 @@ export function ActivityCalendar({ width={cell} height={cell} rx="2.5" - fill={cellColor(day?.value, thresholds)} + fill={cellColor(day, thresholds, finishThresholds)} strokeWidth="1" className="stroke-transparent hover:stroke-[#00000059]" onMouseEnter={(event) => showTooltip(event, text)} diff --git a/apps/frontend/components/Chart.tsx b/apps/frontend/components/Chart.tsx index 3ff81f6..da4c16b 100644 --- a/apps/frontend/components/Chart.tsx +++ b/apps/frontend/components/Chart.tsx @@ -3,7 +3,7 @@ import { useState } from 'react'; import { format } from 'date-fns'; import { DAY_MS, localizeUtcDay } from '@teerank/teerank/date'; -import { formatInteger } from '../utils/format'; +import { formatFinishTime, formatInteger } from '../utils/format'; export type ChartPoint = { date: Date; @@ -220,3 +220,273 @@ export function BarChart({
); } + +export type StepPoint = { + date: Date; + value: number; + label: string; +}; + +const STEP_HEIGHT = 150; +const STEP_INNER_HEIGHT = STEP_HEIGHT - MARGIN.top - MARGIN.bottom; + +export function StepLineChart({ + points, + endDate, + formatValue = formatFinishTime, + fontSize = 14, +}: { + points: StepPoint[]; + endDate?: Date; + formatValue?: (value: number) => string; + fontSize?: number; +}) { + const [tooltip, setTooltip] = useState<{ x: number; y: number; text: string } | null>(null); + + if (points.length < 2) { + return null; + } + + const showTooltip = (event: React.MouseEvent, text: string) => { + const bounds = event.currentTarget.getBoundingClientRect(); + setTooltip({ x: bounds.left + bounds.width / 2, y: bounds.top, text }); + }; + + const hideTooltip = () => { + setTooltip(null); + }; + + const start = points[0].date.getTime(); + const end = Math.max(points[points.length - 1].date.getTime(), endDate?.getTime() ?? 0); + const span = Math.max(1, end - start); + const scaleX = (time: number) => MARGIN.left + ((time - start) / span) * INNER_WIDTH; + + const values = points.map((point) => point.value); + let low = Math.min(...values); + let high = Math.max(...values); + + if (low === high) { + low -= 1; + high += 1; + } + + const pad = (high - low) * 0.08; + const scaleY = (value: number) => + MARGIN.top + + STEP_INNER_HEIGHT * (1 - (value - (low - pad)) / (high - low + 2 * pad)); + + const path = + points + .map((point, index) => { + const x = scaleX(point.date.getTime()); + const y = scaleY(point.value); + return index === 0 ? `M ${x} ${y}` : `H ${x} V ${y}`; + }) + .join(' ') + ` H ${WIDTH - MARGIN.right}`; + + const yearTicks: number[] = []; + + for ( + let year = new Date(start).getUTCFullYear() + 1; + year <= new Date(end).getUTCFullYear(); + year++ + ) { + yearTicks.push(Date.UTC(year, 0, 1)); + } + + const tickStep = Math.max(1, Math.ceil(yearTicks.length / 6)); + const xTicks = + yearTicks.length > 0 + ? yearTicks + .filter((_, index) => index % tickStep === 0) + .map((time) => ({ time, label: format(new Date(time), 'yyyy') })) + : [ + { time: start, label: format(new Date(start), 'MMM yyyy') }, + { time: end, label: format(new Date(end), 'MMM yyyy') }, + ]; + + return ( +
+ + + + {xTicks.map((tick, index) => ( + + {tick.label} + + ))} + + + + {points.map((point) => ( + showTooltip(event, point.label)} + onMouseLeave={hideTooltip} + /> + ))} + + + {tooltip !== null && ( +
+ {tooltip.text} +
+ )} +
+ ); +} + +const STACK_COLORS = [ + '#997700', '#5f7a8c', '#a05a2c', '#4a7c59', + '#8c5f7a', '#c2a14d', '#6b7a4a', '#999999', +]; + +export function StackedBarChart({ + keys, + points, + formatDate, + formatTooltipDate, + formatValue = formatInteger, + emptyLabel = 'No data yet', + fontSize = 14, +}: { + keys: string[]; + points: { date: Date; values: number[] }[]; + formatDate?: (date: Date) => string; + formatTooltipDate?: (date: Date) => string; + formatValue?: (value: number) => string; + emptyLabel?: string; + fontSize?: number; +}) { + const [tooltip, setTooltip] = useState<{ x: number; y: number; lines: string[] } | null>(null); + + if (points.length === 0 || keys.length === 0) { + return ; + } + + const showTooltip = (event: React.MouseEvent, lines: string[]) => { + const bounds = event.currentTarget.getBoundingClientRect(); + setTooltip({ x: bounds.left + bounds.width / 2, y: bounds.top, lines }); + }; + + const hideTooltip = () => { + setTooltip(null); + }; + + const dates = points.map((point) => point.date); + const axisDate = formatDate ?? ((date: Date) => formatUtcDate(date, axisDatePattern(dates))); + const tooltipDate = + formatTooltipDate ?? ((date: Date) => formatUtcDate(date, 'MMM d, yyyy')); + + const totals = points.map((point) => point.values.reduce((sum, value) => sum + value, 0)); + const max = niceCeiling(Math.max(...totals, 1)); + const scaleY = (value: number) => MARGIN.top + INNER_HEIGHT * (1 - value / max); + const barWidth = (INNER_WIDTH / points.length) * 0.7; + + return ( +
+ + + + + {points.map((point, index) => { + const x = slotX(index, points.length) - barWidth / 2; + const lines = [ + tooltipDate(point.date), + ...keys.flatMap((key, keyIndex) => + point.values[keyIndex] > 0 ? `${key}: ${formatValue(point.values[keyIndex])}` : [] + ), + ]; + + let stacked = 0; + + return ( + showTooltip(event, lines)} + onMouseLeave={hideTooltip} + > + {keys.map((key, keyIndex) => { + const value = point.values[keyIndex]; + if (value <= 0) { + return null; + } + const y = scaleY(stacked + value); + const height = INNER_HEIGHT * (value / max); + stacked += value; + return ( + + ); + })} + + ); + })} + + +
+ {keys.map((key, index) => ( + + + {key} + + ))} +
+ + {tooltip !== null && ( +
+ {tooltip.lines.map((line) => ( +
{line}
+ ))} +
+ )} +
+ ); +} diff --git a/apps/frontend/components/DDNetAttribution.tsx b/apps/frontend/components/DDNetAttribution.tsx new file mode 100644 index 0000000..4bf9d50 --- /dev/null +++ b/apps/frontend/components/DDNetAttribution.tsx @@ -0,0 +1,10 @@ +export function DDNetAttribution({ label }: { label?: string }) { + return ( +

+ {label ?? 'Official record data from'}{' '} + + DDNet.org + +

+ ); +} diff --git a/apps/frontend/components/DailyPlayersSection.tsx b/apps/frontend/components/DailyPlayersSection.tsx index 43f3822..d0cde86 100644 --- a/apps/frontend/components/DailyPlayersSection.tsx +++ b/apps/frontend/components/DailyPlayersSection.tsx @@ -1,7 +1,7 @@ 'use client'; import { useState } from 'react'; -import { BarChart } from './Chart'; +import { BarChart, StackedBarChart } from './Chart'; import { fillSeries } from '../utils/series'; import { eachUtcDay, parseUtcDay } from '@teerank/teerank/date'; import { DailyPlayersPayload } from '../utils/dailyPlayers'; @@ -13,16 +13,66 @@ const PRESETS: { key: string; label: string }[] = [ { key: 'all', label: 'All time' }, ]; +const SOURCES: { key: string; label: string }[] = [ + { key: 'teerank', label: 'Teerank players' }, + { key: 'ddnet', label: 'DDNet total' }, + { key: 'bycountry', label: 'DDNet by country' }, + { key: 'bymod', label: 'DDNet by mod' }, +]; + +function Dropdown({ + options, + current, + onSelect, +}: { + options: { key: string; label: string }[]; + current: string; + onSelect: (key: string) => void; +}) { + const [open, setOpen] = useState(false); + + return ( + + + + {open && ( + <> + setOpen(false)} /> + + {options.map((option) => ( + + ))} + + + )} + + ); +} + export function DailyPlayersSection({ initial }: { initial: DailyPlayersPayload }) { const [payload, setPayload] = useState(initial); const [loading, setLoading] = useState(false); - const [open, setOpen] = useState(false); - const select = async (range: string) => { - setOpen(false); + const select = async (range: string, source: string) => { setLoading(true); try { - const response = await fetch(`/api/daily-players?range=${range}`); + const response = await fetch(`/api/daily-players?range=${range}&source=${source}`); if (response.ok) { setPayload(await response.json()); } @@ -31,50 +81,64 @@ export function DailyPlayersSection({ initial }: { initial: DailyPlayersPayload } }; + const stacked = payload.keys !== undefined && payload.stacks !== undefined; const domain = eachUtcDay(parseUtcDay(payload.from), parseUtcDay(payload.to)); return (

Daily players

- - - - {open && ( - <> - setOpen(false)} /> - - {PRESETS.map((preset) => ( - - ))} - - - )} + + select(payload.range, source)} + /> + select(range, payload.source)} + />
- ({ at: parseUtcDay(day), value: players })) - )} - emptyLabel="No history yet — the first rollup lands tomorrow" - /> + {stacked ? ( + ({ + date: parseUtcDay(day), + values, + }))} + emptyLabel="No DDNet history yet — the import is still running" + /> + ) : payload.source === 'teerank' ? ( + ({ at: parseUtcDay(day), value: players })) + )} + emptyLabel="No history yet — the first rollup lands tomorrow" + /> + ) : ( + ({ + date: parseUtcDay(day), + value: players, + }))} + emptyLabel="No DDNet history yet — the import is still running" + /> + )}
+ + {payload.source !== 'teerank' && ( +

+ Average concurrent players on official DDNet servers since Dec 2014 — data from{' '} + + DDNet.org + +

+ )}
); } diff --git a/apps/frontend/components/RecordList.tsx b/apps/frontend/components/RecordList.tsx new file mode 100644 index 0000000..080f61c --- /dev/null +++ b/apps/frontend/components/RecordList.tsx @@ -0,0 +1,199 @@ +import { Fragment, Suspense } from 'react'; +import Link from 'next/link'; +import { formatFinishTime, formatInteger, formatTimeDelta } from '../utils/format'; +import { List, ListCell } from './List'; +import { SharedRatio } from './SharedRatio'; +import { EyeToggle } from './EyeToggle'; +import { encodeString } from '../utils/encoding'; + +export type RecordRanks = 'solo' | 'team' | 'duo'; + +function RanksToggle({ + ranks, + pathname, +}: { + ranks: RecordRanks; + pathname: string; +}) { + const options: { value: RecordRanks; label: string }[] = [ + { value: 'solo', label: 'Solo' }, + { value: 'team', label: 'Team' }, + { value: 'duo', label: 'Duo' }, + ]; + + return ( + + {options.map(({ value, label }) => + value === ranks ? ( + + {label} + + ) : ( + + {label} + + ) + )} + + ); +} + +export function RecordList({ + records, + recordCount, + ranks, + pathname, + showRank, + showMap, + showDelta, + showSharedToggle, +}: { + records: { + rank?: number; + map?: { name: string; gameTypeName: string }; + players: { name: string; pollCount: number; occurrenceCount: number }[]; + time: number; + delta?: number | null; + dateLabel: string; + }[]; + recordCount?: number; + ranks?: RecordRanks; + pathname?: string; + showRank?: boolean; + showMap?: boolean; + showDelta?: boolean; + showSharedToggle?: boolean; +}) { + const columns: { title: React.ReactNode; expand: boolean }[] = []; + + if (showRank) { + columns.push({ + title: '', + expand: false, + }); + } + + if (showMap) { + columns.push({ + title: 'Map', + expand: true, + }); + } + + columns.push({ + title: ( + <> + Name + {ranks !== undefined && pathname !== undefined && ( + + )} + {showSharedToggle && ( + + + + )} + + ), + expand: true, + }); + + columns.push({ + title: 'Time', + expand: false, + }); + + if (showDelta) { + columns.push({ + title: 'Improved by', + expand: false, + }); + } + + columns.push({ + title: 'Date', + expand: false, + }); + + return ( + + {records.map((record, index) => ( + + {showRank && ( + + )} + {showMap && ( + + )} + + {record.players.map((player, playerIndex) => ( + + {playerIndex > 0 && ', '} + + {player.name} + + + + ))} + + + {showDelta && ( + + )} + + + ))} + + ); +} diff --git a/apps/frontend/components/TeammateList.tsx b/apps/frontend/components/TeammateList.tsx deleted file mode 100644 index d714eed..0000000 --- a/apps/frontend/components/TeammateList.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import { Fragment } from 'react'; -import { intervalToDuration } from 'date-fns'; -import { List, ListCell } from './List'; -import { durationClassName, LastSeen } from './LastSeen'; -import { encodeString } from '../utils/encoding'; -import { formatDurationShort, formatPlayTime } from '../utils/format'; -import { Teammate } from '../utils/teammates'; - -export function TeammateList({ - teammates, - pageCount, -}: { - teammates: Teammate[]; - pageCount?: number; -}) { - return ( - - {teammates.map((teammate) => { - const lastTogether = intervalToDuration({ - start: teammate.lastPlayedAt, - end: new Date(), - }); - - return ( - - - - - - - - ); - })} - - ); -} diff --git a/apps/frontend/utils/activity.ts b/apps/frontend/utils/activity.ts index be12d5d..385bf9b 100644 --- a/apps/frontend/utils/activity.ts +++ b/apps/frontend/utils/activity.ts @@ -76,6 +76,23 @@ function tooltipDate(day: Date) { return format(day, 'MMM d, yyyy'); } +function finishSuffix(finishCount: number) { + return finishCount > 0 + ? ` · ${formatInteger(finishCount)} ${finishCount === 1 ? 'finish' : 'finishes'}` + : ''; +} + +function finishOnlyDay(day: Date, finishCount: number): ActivityDay { + return { + day: formatUtcDay(day), + value: finishCount, + kind: 'finishes', + tooltip: + `${formatInteger(finishCount)} ${finishCount === 1 ? 'finish' : 'finishes'}` + + ` on ${tooltipDate(day)} (DDNet records)`, + }; +} + export async function getPlayerActivity(playerId: number, params: RangeParams) { const { range, from, to } = await resolveDomain(params, async () => { const result = await prisma.playerDay.aggregate({ @@ -87,14 +104,21 @@ export async function getPlayerActivity(playerId: number, params: RangeParams) { const rows = await prisma.playerDay.findMany({ where: { playerId, day: { gte: from, lte: to } }, - select: { day: true, playTime: true }, + select: { day: true, playTime: true, finishCount: true }, }); - return payload(range, from, to, rows.map((row) => ({ - day: formatUtcDay(row.day), - value: row.playTime, - tooltip: `${formatPlayTime(BigInt(row.playTime))} on ${tooltipDate(row.day)}`, - }))); + return payload(range, from, to, rows.map((row) => { + if (row.playTime <= 0 && row.finishCount > 0) { + return finishOnlyDay(row.day, row.finishCount); + } + return { + day: formatUtcDay(row.day), + value: row.playTime, + tooltip: + `${formatPlayTime(BigInt(row.playTime))} on ${tooltipDate(row.day)}` + + finishSuffix(row.finishCount), + }; + })); } export async function getClanActivity(clanId: number, params: RangeParams) { @@ -150,14 +174,21 @@ export async function getMapActivity(mapId: number, params: RangeParams) { const rows = await prisma.mapDay.findMany({ where: { mapId, day: { gte: from, lte: to } }, - select: { day: true, playTime: true, playerCount: true }, + select: { day: true, playTime: true, playerCount: true, finishCount: true }, }); - return payload(range, from, to, rows.map((row) => ({ - day: formatUtcDay(row.day), - value: row.playTime, - tooltip: `${formatPlayTime(BigInt(row.playTime))} · ${formatInteger(row.playerCount)} players on ${tooltipDate(row.day)}`, - }))); + return payload(range, from, to, rows.map((row) => { + if (row.playTime <= 0 && row.finishCount > 0) { + return finishOnlyDay(row.day, row.finishCount); + } + return { + day: formatUtcDay(row.day), + value: row.playTime, + tooltip: + `${formatPlayTime(BigInt(row.playTime))} · ${formatInteger(row.playerCount)} players` + + ` on ${tooltipDate(row.day)}` + finishSuffix(row.finishCount), + }; + })); } export async function getServerActivity(gameServerId: number, params: RangeParams) { diff --git a/apps/frontend/utils/dailyPlayers.ts b/apps/frontend/utils/dailyPlayers.ts index 7588f06..a5d9ef2 100644 --- a/apps/frontend/utils/dailyPlayers.ts +++ b/apps/frontend/utils/dailyPlayers.ts @@ -4,51 +4,192 @@ import { addUtcDays, formatUtcDay, utcYesterday } from '@teerank/teerank/date'; export type DailyPlayersPayload = { range: string; + source: string; from: string; to: string; - days: { day: string; players: number }[]; + days?: { day: string; players: number }[]; + keys?: string[]; + stacks?: { day: string; values: number[] }[]; }; const MAX_SPAN_DAYS = 5 * 366; +const DDNET_MAX_SPAN_DAYS = 20 * 366; + const PRESET_DAYS: Record = { '30d': 30, '90d': 90, '1y': 365, }; +export const DAILY_PLAYERS_SOURCES = ['teerank', 'ddnet', 'bycountry', 'bymod'] as const; +export type DailyPlayersSource = typeof DAILY_PLAYERS_SOURCES[number]; + +const TOP_KEY_COUNT = 7; + export const CACHE_SECONDS = 3600; -export const getDailyPlayers = unstable_cache( - async (range: string): Promise => { - const to = utcYesterday(); - let from: Date; - - if (range === 'all') { - const result = await prisma.globalDay.aggregate({ _min: { day: true } }); - const minFrom = addUtcDays(to, -(MAX_SPAN_DAYS - 1)); - const minDay = result._min.day; - from = minDay === null || minDay > to ? to : minDay < minFrom ? minFrom : minDay; - } else { - range = PRESET_DAYS[range] !== undefined ? range : '90d'; - from = addUtcDays(to, -(PRESET_DAYS[range] - 1)); +async function resolveRange( + range: string, + getMinDay: () => Promise, + maxSpanDays: number +) { + const to = utcYesterday(); + let from: Date; + + if (range === 'all') { + const minFrom = addUtcDays(to, -(maxSpanDays - 1)); + const minDay = await getMinDay(); + from = minDay === null || minDay > to ? to : minDay < minFrom ? minFrom : minDay; + } else { + range = PRESET_DAYS[range] !== undefined ? range : '90d'; + from = addUtcDays(to, -(PRESET_DAYS[range] - 1)); + } + + return { range, from, to }; +} + +function spanDaysOf(from: Date, to: Date) { + return (to.getTime() - from.getTime()) / 86_400_000 + 1; +} + +function bucketLabel(day: Date, spanDays: number) { + if (spanDays <= 366) { + return formatUtcDay(day); + } + if (spanDays <= MAX_SPAN_DAYS) { + const weekStart = addUtcDays(day, -((day.getUTCDay() + 6) % 7)); + return formatUtcDay(weekStart); + } + return `${formatUtcDay(day).slice(0, 7)}-01`; +} + +async function getTeerankPayload(range: string): Promise> { + const resolved = await resolveRange(range, async () => { + const result = await prisma.globalDay.aggregate({ _min: { day: true } }); + return result._min.day; + }, MAX_SPAN_DAYS); + + const rows = await prisma.globalDay.findMany({ + where: { day: { gte: resolved.from, lte: resolved.to } }, + select: { day: true, playerCount: true }, + orderBy: { day: 'asc' }, + }); + + return { + range: resolved.range, + from: formatUtcDay(resolved.from), + to: formatUtcDay(resolved.to), + days: rows.map(({ day, playerCount }) => ({ day: formatUtcDay(day), players: playerCount })), + }; +} + +async function getDdnetMinDay(kind: number) { + const result = await prisma.ddnetOnlineDay.aggregate({ + _min: { day: true }, + where: { kind }, + }); + return result._min.day; +} + +async function getDdnetTotalPayload(range: string): Promise> { + const resolved = await resolveRange(range, () => getDdnetMinDay(0), DDNET_MAX_SPAN_DAYS); + + const rows = await prisma.ddnetOnlineDay.findMany({ + where: { kind: 0, key: 'ALL', day: { gte: resolved.from, lte: resolved.to } }, + select: { day: true, avgPlayers: true }, + orderBy: { day: 'asc' }, + }); + + const spanDays = spanDaysOf(resolved.from, resolved.to); + const buckets = new Map(); + + for (const row of rows) { + const label = bucketLabel(row.day, spanDays); + const bucket = buckets.get(label) ?? { sum: 0, count: 0 }; + bucket.sum += row.avgPlayers; + bucket.count += 1; + buckets.set(label, bucket); + } + + return { + range: resolved.range, + from: formatUtcDay(resolved.from), + to: formatUtcDay(resolved.to), + days: [...buckets.entries()].map(([day, { sum, count }]) => ({ + day, + players: Math.round(sum / count), + })), + }; +} + +async function getDdnetBreakdownPayload( + range: string, + kind: number +): Promise> { + const resolved = await resolveRange(range, () => getDdnetMinDay(kind), DDNET_MAX_SPAN_DAYS); + + const rows = await prisma.ddnetOnlineDay.findMany({ + where: { kind, key: { not: 'ALL' }, day: { gte: resolved.from, lte: resolved.to } }, + select: { day: true, key: true, avgPlayers: true }, + orderBy: { day: 'asc' }, + }); + + const totals = new Map(); + for (const row of rows) { + totals.set(row.key, (totals.get(row.key) ?? 0) + row.avgPlayers); + } + + const topKeys = [...totals.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, TOP_KEY_COUNT) + .map(([key]) => key); + const keys = totals.size > topKeys.length ? [...topKeys, 'Other'] : topKeys; + const keyIndex = new Map(keys.map((key, index) => [key, index])); + + const spanDays = spanDaysOf(resolved.from, resolved.to); + const buckets = new Map }>(); + + for (const row of rows) { + const label = bucketLabel(row.day, spanDays); + let bucket = buckets.get(label); + if (bucket === undefined) { + bucket = { sums: keys.map(() => 0), days: new Set() }; + buckets.set(label, bucket); } + const index = keyIndex.get(row.key) ?? keys.length - 1; + bucket.sums[index] += row.avgPlayers; + bucket.days.add(formatUtcDay(row.day)); + } + + return { + range: resolved.range, + from: formatUtcDay(resolved.from), + to: formatUtcDay(resolved.to), + keys, + stacks: [...buckets.entries()].map(([day, bucket]) => ({ + day, + values: bucket.sums.map((sum) => Math.round(sum / bucket.days.size)), + })), + }; +} - const rows = await prisma.globalDay.findMany({ - where: { day: { gte: from, lte: to } }, - select: { day: true, playerCount: true }, - orderBy: { day: 'asc' }, - }); - - return { - range, - from: formatUtcDay(from), - to: formatUtcDay(to), - days: rows.map(({ day, playerCount }) => ({ - day: formatUtcDay(day), - players: playerCount, - })), - }; +export const getDailyPlayers = unstable_cache( + async (range: string, source = 'teerank'): Promise => { + const source_: DailyPlayersSource = (DAILY_PLAYERS_SOURCES as readonly string[]).includes(source) + ? (source as DailyPlayersSource) + : 'teerank'; + + switch (source_) { + case 'ddnet': + return { source: source_, ...(await getDdnetTotalPayload(range)) }; + case 'bycountry': + return { source: source_, ...(await getDdnetBreakdownPayload(range, 0)) }; + case 'bymod': + return { source: source_, ...(await getDdnetBreakdownPayload(range, 1)) }; + default: + return { source: source_, ...(await getTeerankPayload(range)) }; + } }, ['home-daily-players'], { revalidate: CACHE_SECONDS } diff --git a/apps/frontend/utils/ddnetMap.ts b/apps/frontend/utils/ddnetMap.ts new file mode 100644 index 0000000..bdccc64 --- /dev/null +++ b/apps/frontend/utils/ddnetMap.ts @@ -0,0 +1,508 @@ +import { STUB_MIN_POLL_COUNT, STUB_OCCURRENCE_RATIO } from '@teerank/teerank'; +import { + countDdnetMapRecordsWithoutShared, + listDdnetMapRecordsWithoutShared, +} from '@prisma/client/sql'; +import prisma from './prisma'; + +export type RecordPlayer = { + name: string; + pollCount: number; + occurrenceCount: number; +}; + +export async function getDdnetMap(mapId: number) { + return prisma.ddnetMap.findUnique({ + where: { + mapId, + }, + }); +} + +export async function getDdnetMapHeader(mapId: number) { + const ddnetMap = await getDdnetMap(mapId); + + if (ddnetMap === null) { + return null; + } + + const [mappers, thumb] = await Promise.all([ + prisma.ddnetMapMapper.findMany({ + where: { + mapId, + }, + orderBy: { + mapperName: 'asc', + }, + }), + prisma.ddnetMapThumb.findUnique({ + select: { + mapId: true, + }, + where: { + mapId, + }, + }), + ]); + + return { + ddnetMap, + mappers: mappers.map((mapper) => mapper.mapperName), + hasThumb: thumb !== null, + }; +} + +async function playersById(playerIds: number[]) { + const players = await prisma.player.findMany({ + select: { + id: true, + name: true, + pollCount: true, + occurrenceCount: true, + }, + where: { + id: { + in: [...new Set(playerIds)], + }, + }, + }); + + const byId = new Map(players.map((player) => [player.id, player])); + + return (playerId: number): RecordPlayer => + byId.get(playerId) ?? { + name: `#${playerId}`, + pollCount: 0, + occurrenceCount: 0, + }; +} + +export async function listSoloRecords({ + mapId, + page, + hideShared, +}: { + mapId: number; + page: number; + hideShared: boolean; +}) { + const take = 100; + const skip = (page - 1) * take; + + if (hideShared) { + const [rows, counts] = await Promise.all([ + prisma.$queryRawTyped(listDdnetMapRecordsWithoutShared( + mapId, STUB_MIN_POLL_COUNT, STUB_OCCURRENCE_RATIO, take, skip + )), + prisma.$queryRawTyped(countDdnetMapRecordsWithoutShared( + mapId, STUB_MIN_POLL_COUNT, STUB_OCCURRENCE_RATIO + )), + ]); + + return { + count: Number(counts[0]?.count ?? 0), + records: rows.map((row, index) => ({ + rank: skip + index + 1, + players: [ + { + name: row.name, + pollCount: row.pollCount, + occurrenceCount: row.occurrenceCount, + }, + ], + time: row.time, + date: row.bestAt, + })), + }; + } + + const [bests, count] = await Promise.all([ + prisma.ddnetRaceBest.findMany({ + where: { + mapId, + }, + orderBy: [ + { + time: 'asc', + }, + { + playerId: 'asc', + }, + ], + take, + skip, + }), + prisma.ddnetRaceBest.count({ + where: { + mapId, + }, + }), + ]); + + const playerFor = await playersById(bests.map((best) => best.playerId)); + + return { + count, + records: bests.map((best, index) => ({ + rank: skip + index + 1, + players: [playerFor(best.playerId)], + time: best.time, + date: best.bestAt, + })), + }; +} + +export async function listTeamRecords({ + mapId, + page, + size, +}: { + mapId: number; + page: number; + size?: number; +}) { + const take = 100; + const skip = (page - 1) * take; + const where = { + mapId, + size, + }; + + const [races, count] = await Promise.all([ + prisma.ddnetTeamRace.findMany({ + where, + orderBy: [ + { + time: 'asc', + }, + { + timestamp: 'asc', + }, + ], + take, + skip, + }), + prisma.ddnetTeamRace.count({ + where, + }), + ]); + + const rosterRows = await prisma.ddnetTeamRacePlayer.findMany({ + where: { + teamId: { + in: races.map((race) => race.teamId), + }, + }, + }); + + const playerFor = await playersById(rosterRows.map((row) => row.playerId)); + + const rosters = new Map(); + + for (const row of rosterRows) { + const key = Buffer.from(row.teamId).toString('hex'); + const roster = rosters.get(key) ?? []; + roster.push(playerFor(row.playerId)); + rosters.set(key, roster); + } + + return { + count, + records: races.map((race, index) => ({ + rank: skip + index + 1, + players: ( + rosters.get(Buffer.from(race.teamId).toString('hex')) ?? [] + ).sort((a, b) => a.name.localeCompare(b.name)), + time: race.time, + date: race.timestamp, + })), + }; +} + +export async function getRecordEvents(mapId: number) { + const rows = await prisma.ddnetRecord.findMany({ + where: { + mapId, + }, + orderBy: [ + { + timestamp: 'asc', + }, + { + playerId: 'asc', + }, + ], + }); + + const playerFor = await playersById(rows.map((row) => row.playerId)); + + const events: { date: Date; time: number; names: string[] }[] = []; + + for (const row of rows) { + const last = events[events.length - 1]; + + if ( + last !== undefined && + last.date.getTime() === row.timestamp.getTime() && + last.time === row.time + ) { + last.names.push(playerFor(row.playerId).name); + } else { + events.push({ + date: row.timestamp, + time: row.time, + names: [playerFor(row.playerId).name], + }); + } + } + + return events; +} + +type FeedEvent = { + mapId: number; + date: Date; + time: number; + playerIds: number[]; +}; + +function groupFeedEvents( + rows: { mapId: number; timestamp: Date; playerId: number; time: number }[] +) { + const events: FeedEvent[] = []; + const byKey = new Map(); + + for (const row of rows) { + const key = `${row.mapId}\0${row.timestamp.getTime()}\0${row.time}`; + const existing = byKey.get(key); + + if (existing !== undefined) { + existing.playerIds.push(row.playerId); + } else { + const event = { + mapId: row.mapId, + date: row.timestamp, + time: row.time, + playerIds: [row.playerId], + }; + byKey.set(key, event); + events.push(event); + } + } + + return events; +} + +export async function listRecentRecords() { + const since = new Date(Date.now() - 7 * 24 * 3600 * 1000); + const orderBy = [ + { + timestamp: 'desc' as const, + }, + { + mapId: 'asc' as const, + }, + ]; + + let events = groupFeedEvents( + await prisma.ddnetRecord.findMany({ + where: { + timestamp: { + gte: since, + }, + }, + orderBy, + }) + ); + + if (events.length < 10) { + events = groupFeedEvents( + await prisma.ddnetRecord.findMany({ + orderBy, + take: 200, + }) + ).slice(0, 50); + } + + const mapIds = [...new Set(events.map((event) => event.mapId))]; + + const [maps, playerFor, history] = await Promise.all([ + prisma.map.findMany({ + select: { + id: true, + name: true, + gameTypeName: true, + }, + where: { + id: { + in: mapIds, + }, + }, + }), + playersById(events.flatMap((event) => event.playerIds)), + prisma.ddnetRecord.findMany({ + select: { + mapId: true, + timestamp: true, + time: true, + }, + where: { + mapId: { + in: mapIds, + }, + }, + orderBy: { + timestamp: 'asc', + }, + }), + ]); + + const mapsById = new Map(maps.map((map) => [map.id, map])); + + const historyByMap = new Map(); + + for (const row of history) { + const mapHistory = historyByMap.get(row.mapId) ?? []; + mapHistory.push(row); + historyByMap.set(row.mapId, mapHistory); + } + + return events.flatMap((event) => { + const map = mapsById.get(event.mapId); + + if (map === undefined) { + return []; + } + + let previousTime: number | null = null; + + for (const row of historyByMap.get(event.mapId) ?? []) { + if (row.timestamp.getTime() >= event.date.getTime()) { + break; + } + previousTime = row.time; + } + + return [ + { + map: { + name: map.name, + gameTypeName: map.gameTypeName, + }, + players: event.playerIds + .map(playerFor) + .sort((a, b) => a.name.localeCompare(b.name)), + time: event.time, + delta: previousTime === null ? null : event.time - previousTime, + date: event.date, + }, + ]; + }); +} + +export async function getCompareSplits(mapId: number, playerName: string) { + const player = await prisma.player.findUnique({ + select: { + id: true, + }, + where: { + name: playerName, + }, + }); + + if (player === null) { + return null; + } + + const best = await prisma.ddnetRaceBest.findUnique({ + select: { + splits: true, + }, + where: { + mapId_playerId: { + mapId, + playerId: player.id, + }, + }, + }); + + if (best === null || best.splits.length === 0) { + return null; + } + + return best.splits; +} + +export async function getMapperMaps(mapperName: string) { + const mapperRows = await prisma.ddnetMapMapper.findMany({ + where: { + mapperName, + }, + }); + + if (mapperRows.length === 0) { + return null; + } + + const mapIds = mapperRows.map((row) => row.mapId); + + const [ddnetMaps, maps] = await Promise.all([ + prisma.ddnetMap.findMany({ + where: { + mapId: { + in: mapIds, + }, + }, + }), + prisma.map.findMany({ + select: { + id: true, + name: true, + gameTypeName: true, + }, + where: { + id: { + in: mapIds, + }, + }, + }), + ]); + + const mapsById = new Map(maps.map((map) => [map.id, map])); + + return ddnetMaps + .flatMap((ddnetMap) => { + const map = mapsById.get(ddnetMap.mapId); + + if (map === undefined) { + return []; + } + + return [ + { + name: map.name, + gameTypeName: map.gameTypeName, + category: ddnetMap.category, + releasedAt: ddnetMap.releasedAt, + points: ddnetMap.points, + finishCount: ddnetMap.finishCount, + medianTime: ddnetMap.medianTime, + }, + ]; + }) + .sort((a, b) => { + if (a.releasedAt === null && b.releasedAt === null) { + return a.name.localeCompare(b.name); + } + if (a.releasedAt === null) { + return 1; + } + if (b.releasedAt === null) { + return -1; + } + return ( + b.releasedAt.getTime() - a.releasedAt.getTime() || + a.name.localeCompare(b.name) + ); + }); +} diff --git a/apps/frontend/utils/ddnetPlayer.ts b/apps/frontend/utils/ddnetPlayer.ts new file mode 100644 index 0000000..944de2f --- /dev/null +++ b/apps/frontend/utils/ddnetPlayer.ts @@ -0,0 +1,313 @@ +import { format } from 'date-fns'; +import prisma from './prisma'; + +const TIER_ORDER = ['Novice', 'Moderate', 'Brutal', 'Insane']; + +// @db.Date values come back as UTC midnight; re-anchor before formatting. +export function formatUtcDate(date: Date, pattern = 'MMM d, yyyy') { + return format( + new Date(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()), + pattern + ); +} + +export function getDdnetPlayer(playerId: number) { + return prisma.ddnetPlayer.findUnique({ + where: { playerId }, + }); +} + +export type DdnetJourneyTier = { + category: string; + firstMapName: string; + firstFinishAt: Date; + finishCount: number; + points: number; +}; + +export type DdnetJourney = { + tiers: DdnetJourneyTier[]; + firstMapName: string | null; + lastMapName: string | null; +}; + +export async function getDdnetJourney( + playerId: number, + lastFinishAt: Date +): Promise { + const bests = await prisma.ddnetRaceBest.findMany({ + where: { playerId }, + select: { mapId: true, finishCount: true, firstFinishAt: true, bestAt: true }, + orderBy: [{ firstFinishAt: 'asc' }, { mapId: 'asc' }], + }); + + const mapIds = bests.map((best) => best.mapId); + + const [maps, ddnetMaps] = await Promise.all([ + prisma.map.findMany({ + where: { id: { in: mapIds } }, + select: { id: true, name: true }, + }), + prisma.ddnetMap.findMany({ + where: { mapId: { in: mapIds } }, + select: { mapId: true, category: true, points: true }, + }), + ]); + + const mapNames = new Map(maps.map((map) => [map.id, map.name])); + const ddnetMapsById = new Map(ddnetMaps.map((map) => [map.mapId, map])); + + const tiersByCategory = new Map(); + + for (const best of bests) { + const ddnetMap = ddnetMapsById.get(best.mapId); + + if (ddnetMap === undefined) { + continue; + } + + const tier = tiersByCategory.get(ddnetMap.category); + + if (tier === undefined) { + tiersByCategory.set(ddnetMap.category, { + category: ddnetMap.category, + firstMapName: mapNames.get(best.mapId) ?? '', + firstFinishAt: best.firstFinishAt, + finishCount: best.finishCount, + points: ddnetMap.points, + }); + } else { + tier.finishCount += best.finishCount; + tier.points += ddnetMap.points; + } + } + + const tiers = [ + ...TIER_ORDER.flatMap((category) => tiersByCategory.get(category) ?? []), + ...[...tiersByCategory.values()].filter( + (tier) => !TIER_ORDER.includes(tier.category) + ), + ]; + + const lastDay = lastFinishAt.toISOString().slice(0, 10); + const lastBest = bests.find( + (best) => best.bestAt.toISOString().slice(0, 10) === lastDay + ); + + return { + tiers, + firstMapName: bests.length > 0 ? mapNames.get(bests[0].mapId) ?? null : null, + lastMapName: lastBest === undefined ? null : mapNames.get(lastBest.mapId) ?? null, + }; +} + +export type DdnetBestTime = { + mapId: number; + mapName: string | null; + category: string | null; + time: number; + rank: number; + topPercent: number | null; + bestAt: Date; + hasSplits: boolean; +}; + +function topPercent(timeQuantiles: number[], time: number): number | null { + if (timeQuantiles.length === 0) { + return null; + } + + const index = timeQuantiles.findIndex((quantile) => quantile >= time); + + return index === -1 ? 100 : Math.max(index, 1); +} + +async function getRaceRanks(rows: { mapId: number; time: number }[]) { + if (rows.length === 0) { + return new Map(); + } + + const ranks = await prisma.$queryRaw<{ mapId: number; rank: number }[]>` + SELECT r."mapId", (COUNT(b."playerId") + 1)::int AS "rank" + FROM unnest( + ${rows.map((row) => row.mapId)}::int[], + ${rows.map((row) => row.time)}::float8[] + ) AS r("mapId", "time") + LEFT JOIN "DdnetRaceBest" b + ON b."mapId" = r."mapId" AND b."time" < r."time" + GROUP BY r."mapId" + `; + + return new Map(ranks.map((row) => [row.mapId, row.rank])); +} + +export function countDdnetBestTimes(playerId: number) { + return prisma.ddnetRaceBest.count({ where: { playerId } }); +} + +export async function getDdnetBestTimes( + playerId: number, + { skip, take }: { skip: number; take: number } +): Promise { + const bests = await prisma.ddnetRaceBest.findMany({ + where: { playerId }, + orderBy: [{ bestAt: 'desc' }, { mapId: 'asc' }], + skip, + take, + }); + + const mapIds = bests.map((best) => best.mapId); + + const [maps, ddnetMaps, ranks] = await Promise.all([ + prisma.map.findMany({ + where: { id: { in: mapIds } }, + select: { id: true, name: true }, + }), + prisma.ddnetMap.findMany({ + where: { mapId: { in: mapIds } }, + select: { mapId: true, category: true, timeQuantiles: true }, + }), + getRaceRanks(bests), + ]); + + const mapNames = new Map(maps.map((map) => [map.id, map.name])); + const ddnetMapsById = new Map(ddnetMaps.map((map) => [map.mapId, map])); + + return bests.map((best) => { + const ddnetMap = ddnetMapsById.get(best.mapId); + + return { + mapId: best.mapId, + mapName: mapNames.get(best.mapId) ?? null, + category: ddnetMap?.category ?? null, + time: best.time, + rank: ranks.get(best.mapId) ?? 1, + topPercent: + ddnetMap === undefined ? null : topPercent(ddnetMap.timeQuantiles, best.time), + bestAt: best.bestAt, + hasSplits: best.splits.length > 0, + }; + }); +} + +export type DdnetPartnerRow = { + name: string; + pollCount: number; + occurrenceCount: number; + finishCount: number; + bestTime: number; + bestMapName: string | null; + lastFinishAt: Date; +}; + +export function countDdnetPartners(playerId: number) { + return prisma.ddnetPartner.count({ where: { playerId } }); +} + +export async function getDdnetPartners( + playerId: number, + { skip, take }: { skip: number; take: number } +): Promise { + const partners = await prisma.ddnetPartner.findMany({ + where: { playerId }, + orderBy: [{ finishCount: 'desc' }, { partnerId: 'asc' }], + skip, + take, + }); + + const [players, maps] = await Promise.all([ + prisma.player.findMany({ + where: { id: { in: partners.map((partner) => partner.partnerId) } }, + select: { id: true, name: true, pollCount: true, occurrenceCount: true }, + }), + prisma.map.findMany({ + where: { id: { in: partners.map((partner) => partner.bestMapId) } }, + select: { id: true, name: true }, + }), + ]); + + const playersById = new Map(players.map((player) => [player.id, player])); + const mapNames = new Map(maps.map((map) => [map.id, map.name])); + + return partners.flatMap((partner) => { + const player = playersById.get(partner.partnerId); + + if (player === undefined) { + return []; + } + + return { + name: player.name, + pollCount: player.pollCount, + occurrenceCount: player.occurrenceCount, + finishCount: partner.finishCount, + bestTime: partner.bestTime, + bestMapName: mapNames.get(partner.bestMapId) ?? null, + lastFinishAt: partner.lastFinishAt, + }; + }); +} + +export type DdnetPartnerMapRow = { + mapId: number; + mapName: string | null; + bestTime: number; + finishCount: number; + duoRank: number; + lastFinishAt: Date; +}; + +async function getDuoRanks(rows: { mapId: number; time: number }[]) { + if (rows.length === 0) { + return new Map(); + } + + const ranks = await prisma.$queryRaw<{ mapId: number; rank: number }[]>` + SELECT r."mapId", (COUNT(t."teamId") + 1)::int AS "rank" + FROM unnest( + ${rows.map((row) => row.mapId)}::int[], + ${rows.map((row) => row.time)}::float8[] + ) AS r("mapId", "time") + LEFT JOIN "DdnetTeamRace" t + ON t."mapId" = r."mapId" AND t."size" = 2 AND t."time" < r."time" + GROUP BY r."mapId" + `; + + return new Map(ranks.map((row) => [row.mapId, row.rank])); +} + +export function countDdnetPartnerMaps(playerId: number, partnerId: number) { + return prisma.ddnetPartnerMap.count({ where: { playerId, partnerId } }); +} + +export async function getDdnetPartnerMaps( + playerId: number, + partnerId: number, + { skip, take }: { skip: number; take: number } +): Promise { + const rows = await prisma.ddnetPartnerMap.findMany({ + where: { playerId, partnerId }, + orderBy: [{ lastFinishAt: 'desc' }, { mapId: 'asc' }], + skip, + take, + }); + + const [maps, duoRanks] = await Promise.all([ + prisma.map.findMany({ + where: { id: { in: rows.map((row) => row.mapId) } }, + select: { id: true, name: true }, + }), + getDuoRanks(rows.map((row) => ({ mapId: row.mapId, time: row.bestTime }))), + ]); + + const mapNames = new Map(maps.map((map) => [map.id, map.name])); + + return rows.map((row) => ({ + mapId: row.mapId, + mapName: mapNames.get(row.mapId) ?? null, + bestTime: row.bestTime, + finishCount: row.finishCount, + duoRank: duoRanks.get(row.mapId) ?? 1, + lastFinishAt: row.lastFinishAt, + })); +} diff --git a/apps/frontend/utils/format.ts b/apps/frontend/utils/format.ts index d262823..2fca80c 100644 --- a/apps/frontend/utils/format.ts +++ b/apps/frontend/utils/format.ts @@ -66,6 +66,26 @@ export function formatDurationShort(duration: Duration) { } } +// DDNet finish times: "03:42.86", "1:02:15.40". +export function formatFinishTime(seconds: number) { + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const rest = seconds % 60; + + const pad = (value: number) => value.toString().padStart(2, '0'); + const restFormatted = rest.toFixed(2).padStart(5, '0'); + + if (hours > 0) { + return `${hours}:${pad(minutes)}:${restFormatted}`; + } + return `${pad(minutes)}:${restFormatted}`; +} + +export function formatTimeDelta(seconds: number) { + const sign = seconds >= 0 ? '+' : '-'; + return `${sign}${formatFinishTime(Math.abs(seconds))}`; +} + export function formatInteger(number: number) { return Intl.NumberFormat('en-US', { maximumFractionDigits: 0, diff --git a/apps/scheduler/src/main.ts b/apps/scheduler/src/main.ts index 3e737eb..b24360f 100644 --- a/apps/scheduler/src/main.ts +++ b/apps/scheduler/src/main.ts @@ -10,6 +10,9 @@ import { updateGlobalCountsScheduler } from './schedulers/updateGlobalCountsSche import { archiveSnapshotsScheduler } from './schedulers/archiveSnapshotsScheduler'; import { rollupDayScheduler } from './schedulers/rollupDayScheduler'; import { rollupBackfillScheduler } from './schedulers/rollupBackfillScheduler'; +import { ddnetBackfillScheduler } from './schedulers/ddnetBackfillScheduler'; +import { ddnetImportScheduler } from './schedulers/ddnetImportScheduler'; +import { ddnetOnlineScheduler } from './schedulers/ddnetOnlineScheduler'; async function main() { if (process.env.NODE_ENV === 'development') { @@ -29,6 +32,9 @@ async function main() { archiveSnapshotsScheduler(); rollupDayScheduler(); rollupBackfillScheduler(); + ddnetBackfillScheduler(); + ddnetImportScheduler(); + ddnetOnlineScheduler(); } main(); diff --git a/apps/scheduler/src/schedulers/ddnetBackfillScheduler.ts b/apps/scheduler/src/schedulers/ddnetBackfillScheduler.ts new file mode 100644 index 0000000..e0638e2 --- /dev/null +++ b/apps/scheduler/src/schedulers/ddnetBackfillScheduler.ts @@ -0,0 +1,9 @@ +import { minutesToMilliseconds } from "date-fns"; +import { scheduleDdnetBackfill } from "@teerank/teerank"; +import { schedule } from "../utils"; + +export function ddnetBackfillScheduler() { + schedule(minutesToMilliseconds(15), async () => { + await scheduleDdnetBackfill(); + }); +} diff --git a/apps/scheduler/src/schedulers/ddnetImportScheduler.ts b/apps/scheduler/src/schedulers/ddnetImportScheduler.ts new file mode 100644 index 0000000..446bec6 --- /dev/null +++ b/apps/scheduler/src/schedulers/ddnetImportScheduler.ts @@ -0,0 +1,9 @@ +import { hoursToMilliseconds } from "date-fns"; +import { scheduleDdnetImport } from "@teerank/teerank"; +import { schedule } from "../utils"; + +export function ddnetImportScheduler() { + schedule(hoursToMilliseconds(1), async () => { + await scheduleDdnetImport(); + }); +} diff --git a/apps/scheduler/src/schedulers/ddnetOnlineScheduler.ts b/apps/scheduler/src/schedulers/ddnetOnlineScheduler.ts new file mode 100644 index 0000000..0ab25ea --- /dev/null +++ b/apps/scheduler/src/schedulers/ddnetOnlineScheduler.ts @@ -0,0 +1,9 @@ +import { hoursToMilliseconds } from "date-fns"; +import { scheduleDdnetOnline } from "@teerank/teerank"; +import { schedule } from "../utils"; + +export function ddnetOnlineScheduler() { + schedule(hoursToMilliseconds(1), async () => { + await scheduleDdnetOnline(); + }); +} diff --git a/apps/worker/src/ddnet/applyChunk.ts b/apps/worker/src/ddnet/applyChunk.ts new file mode 100644 index 0000000..4f1b330 --- /dev/null +++ b/apps/worker/src/ddnet/applyChunk.ts @@ -0,0 +1,480 @@ +import { chunk } from "lodash"; +import { minutesToMilliseconds } from "date-fns"; +import { Prisma } from "@prisma/client"; +import { + createRollupPartition, + incrementDdnetPlayers, + incrementMapDayFinishes, + incrementPlayerDayFinishes, + insertDdnetRecords, + listDdnetCurrentRecords, + setMapDayFinishes, + setPlayerDayFinishes, + upsertDdnetPartnerMaps, + upsertDdnetPartners, + upsertDdnetPlayerNames, + upsertDdnetRaceBests, + upsertDdnetTeamRacePlayers, + upsertDdnetTeamRaces, +} from "@prisma/client/sql"; +import { isStubName } from "@teerank/teerank"; +import { prisma } from "../prisma"; +import { DdnetRaceRow, DdnetTeamRaceRow } from "./stream"; + +export const DDNET_GAME_TYPE = 'DDraceNetwork'; + +const LOOKUP_CHUNK_SIZE = 5_000; +const INSERT_CHUNK_SIZE = 2_000; + +export type TransactionClient = Prisma.TransactionClient; + +export type RecordState = Map; + +export type RecordCandidate = { + mapId: number; + playerId: number; + time: number; + timestamp: string; // ISO +}; + +export async function loadRecordState(): Promise { + const rows = await prisma.$queryRawTyped(listDdnetCurrentRecords()); + return new Map(rows.map((row) => [row.mapId, row.time])); +} + +export async function ensureFinishPartitions(ym: string) { + const from = new Date(`${ym}-01T00:00:00Z`); + const to = new Date(Date.UTC(from.getUTCFullYear(), from.getUTCMonth() + 1, 1)); + + for (const table of ['PlayerDay', 'MapDay']) { + await prisma.$queryRawTyped(createRollupPartition(table, from, to)); + } +} + +function utcDay(date: Date) { + return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); +} + +export async function resolveMapIds(mapNames: string[]) { + const distinct = [...new Set(mapNames)]; + const ids = new Map(); + + for (const names of chunk(distinct, LOOKUP_CHUNK_SIZE)) { + const found = await prisma.map.findMany({ + where: { gameTypeName: DDNET_GAME_TYPE, name: { in: names } }, + select: { name: true, id: true }, + }); + for (const map of found) { + ids.set(map.name, map.id); + } + + const missing = names.filter((name) => !ids.has(name)); + if (missing.length > 0) { + await prisma.gameType.upsert({ + where: { name: DDNET_GAME_TYPE }, + create: { name: DDNET_GAME_TYPE, rankMethod: 'TIME' }, + update: {}, + }); + await prisma.map.createMany({ + data: missing.map((name) => ({ name, gameTypeName: DDNET_GAME_TYPE })), + skipDuplicates: true, + }); + const created = await prisma.map.findMany({ + where: { gameTypeName: DDNET_GAME_TYPE, name: { in: missing } }, + select: { name: true, id: true }, + }); + for (const map of created) { + ids.set(map.name, map.id); + } + } + } + + return ids; +} + +async function resolvePlayerIds(lastSeenByName: Map) { + const names = [...lastSeenByName.keys()]; + + for (const names_ of chunk(names, INSERT_CHUNK_SIZE)) { + await prisma.$queryRawTyped( + upsertDdnetPlayerNames(names_, names_.map((name) => lastSeenByName.get(name)!)) + ); + } + + const ids = new Map(); + const stubIds = new Set(); + + for (const names_ of chunk(names, LOOKUP_CHUNK_SIZE)) { + const players = await prisma.player.findMany({ + where: { name: { in: names_ } }, + select: { name: true, id: true, pollCount: true, occurrenceCount: true }, + }); + for (const player of players) { + ids.set(player.name, player.id); + if (isStubName(player.pollCount, player.occurrenceCount)) { + stubIds.add(player.id); + } + } + } + + return { ids, stubIds }; +} + +type BestAccumulator = { + mapId: number; + playerId: number; + time: number; + finishCount: number; + firstFinishAt: string; + bestAt: string; + splits: number[]; +}; + +export async function applyRaceRows( + rows: DdnetRaceRow[], + recordState: RecordState, + options: { + dayCounts: 'increment' | 'set'; + newRows?: (row: DdnetRaceRow) => boolean; + writeState: (tx: TransactionClient, candidates: RecordCandidate[]) => Promise; + } +) { + const isNew = options.newRows ?? (() => true); + + const lastSeenByName = new Map(); + for (const row of rows) { + const seen = lastSeenByName.get(row.playerName); + if (seen === undefined || row.timestamp > seen) { + lastSeenByName.set(row.playerName, row.timestamp); + } + } + + const mapIds = await resolveMapIds(rows.map((row) => row.mapName)); + const { ids: playerIds } = await resolvePlayerIds(lastSeenByName); + + const bests = new Map(); + const playerDays = new Map(); + const mapDays = new Map(); + const players = new Map(); + const candidates: RecordCandidate[] = []; + + const sorted = [...rows].sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); + const runningRecord = new Map(); + + for (const row of sorted) { + const mapId = mapIds.get(row.mapName)!; + const playerId = playerIds.get(row.playerName); + + if (playerId === undefined) { + continue; + } + + const day = utcDay(row.timestamp); + const dayLabel = day.toISOString().slice(0, 10); + + const playerDayKey = `${dayLabel}:${playerId}`; + const playerDay = playerDays.get(playerDayKey); + if (playerDay === undefined) { + playerDays.set(playerDayKey, { day, playerId, count: 1 }); + } else { + playerDay.count += 1; + } + + const mapDayKey = `${dayLabel}:${mapId}`; + const mapDay = mapDays.get(mapDayKey); + if (mapDay === undefined) { + mapDays.set(mapDayKey, { day, mapId, count: 1 }); + } else { + mapDay.count += 1; + } + + if (!isNew(row)) { + continue; + } + + const bestKey = `${mapId}:${playerId}`; + const best = bests.get(bestKey); + if (best === undefined) { + bests.set(bestKey, { + mapId, + playerId, + time: row.time, + finishCount: 1, + firstFinishAt: dayLabel, + bestAt: dayLabel, + splits: row.splits, + }); + } else { + best.finishCount += 1; + if (row.time < best.time) { + best.time = row.time; + best.bestAt = dayLabel; + best.splits = row.splits; + } + } + + const player = players.get(playerId); + if (player === undefined) { + players.set(playerId, { count: 1, first: row.timestamp, last: row.timestamp }); + } else { + player.count += 1; + if (row.timestamp < player.first) player.first = row.timestamp; + if (row.timestamp > player.last) player.last = row.timestamp; + } + + const record = runningRecord.get(mapId) ?? recordState.get(mapId); + if (record === undefined || row.time < record) { + runningRecord.set(mapId, row.time); + candidates.push({ + mapId, + playerId, + time: row.time, + timestamp: row.timestamp.toISOString(), + }); + } + } + + const bestRows = [...bests.values()].sort( + (a, b) => a.mapId - b.mapId || a.playerId - b.playerId + ); + const playerDayRows = [...playerDays.values()].sort( + (a, b) => a.day.getTime() - b.day.getTime() || a.playerId - b.playerId + ); + const mapDayRows = [...mapDays.values()].sort( + (a, b) => a.day.getTime() - b.day.getTime() || a.mapId - b.mapId + ); + const playerRows = [...players.entries()] + .map(([playerId, agg]) => ({ playerId, ...agg })) + .sort((a, b) => a.playerId - b.playerId); + + await prisma.$transaction( + async (tx) => { + for (const rows_ of chunk(bestRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(upsertDdnetRaceBests(JSON.stringify(rows_))); + } + + const dayCountSql = { + increment: { player: incrementPlayerDayFinishes, map: incrementMapDayFinishes }, + set: { player: setPlayerDayFinishes, map: setMapDayFinishes }, + }[options.dayCounts]; + + for (const rows_ of chunk(playerDayRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(dayCountSql.player( + rows_.map((row) => row.day), + rows_.map((row) => row.playerId), + rows_.map((row) => row.count) + )); + } + for (const rows_ of chunk(mapDayRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(dayCountSql.map( + rows_.map((row) => row.day), + rows_.map((row) => row.mapId), + rows_.map((row) => row.count) + )); + } + for (const rows_ of chunk(playerRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(incrementDdnetPlayers( + rows_.map((row) => row.playerId), + rows_.map((row) => row.count), + rows_.map((row) => row.first), + rows_.map((row) => row.last) + )); + } + + await options.writeState(tx, candidates); + }, + { timeout: minutesToMilliseconds(5), maxWait: minutesToMilliseconds(1) } + ); + + return { + playerIds: playerRows.map((row) => row.playerId), + mapIds: [...new Set(bestRows.map((row) => row.mapId))], + }; +} + +export async function replayRecordCandidates( + candidates: RecordCandidate[], + recordState: RecordState, + tx: TransactionClient +) { + const events: RecordCandidate[] = []; + const sorted = [...candidates].sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + + for (const candidate of sorted) { + const record = recordState.get(candidate.mapId); + if (record === undefined || candidate.time < record) { + recordState.set(candidate.mapId, candidate.time); + events.push(candidate); + } + } + + for (const events_ of chunk(events, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(insertDdnetRecords( + events_.map((event) => event.mapId), + events_.map((event) => new Date(event.timestamp)), + events_.map((event) => event.playerId), + events_.map((event) => event.time) + )); + } + + return events.length; +} + +export type TeamRun = { + teamId: string; // hex + mapName: string; + time: number; + timestamp: Date; + playerNames: string[]; +}; + +export function groupTeamRows(rows: DdnetTeamRaceRow[]): TeamRun[] { + const teams = new Map(); + + for (const row of rows) { + const team = teams.get(row.teamId); + if (team === undefined) { + teams.set(row.teamId, { + teamId: row.teamId, + mapName: row.mapName, + time: row.time, + timestamp: row.timestamp, + playerNames: [row.playerName], + }); + } else if (!team.playerNames.includes(row.playerName)) { + team.playerNames.push(row.playerName); + } + } + + return [...teams.values()]; +} + +export async function applyTeamRuns( + teams: TeamRun[], + options: { writeState: (tx: TransactionClient) => Promise } +) { + const lastSeenByName = new Map(); + for (const team of teams) { + for (const name of team.playerNames) { + const seen = lastSeenByName.get(name); + if (seen === undefined || team.timestamp > seen) { + lastSeenByName.set(name, team.timestamp); + } + } + } + + const mapIds = await resolveMapIds(teams.map((team) => team.mapName)); + const { ids: playerIds, stubIds } = await resolvePlayerIds(lastSeenByName); + + const teamRows: { teamId: string; mapId: number; time: number; timestamp: string; size: number }[] = []; + const rosterRows: { teamId: string; playerId: number }[] = []; + const partners = new Map(); + const partnerMaps = new Map(); + + for (const team of teams) { + const mapId = mapIds.get(team.mapName)!; + const memberIds = team.playerNames + .map((name) => playerIds.get(name)) + .filter((id): id is number => id !== undefined); + + teamRows.push({ + teamId: team.teamId, + mapId, + time: team.time, + timestamp: team.timestamp.toISOString(), + size: memberIds.length, + }); + for (const playerId of memberIds) { + rosterRows.push({ teamId: team.teamId, playerId }); + } + + const day = utcDay(team.timestamp); + + // A stub name is many people, so pairs involving one are meaningless. + const pairIds = memberIds.filter((id) => !stubIds.has(id)); + + for (const playerId of pairIds) { + for (const partnerId of pairIds) { + if (playerId === partnerId) { + continue; + } + + const key = `${playerId}:${partnerId}`; + const partner = partners.get(key); + if (partner === undefined) { + partners.set(key, { + playerId, partnerId, finishCount: 1, + bestTime: team.time, bestMapId: mapId, lastFinishAt: day, + }); + } else { + partner.finishCount += 1; + if (team.time < partner.bestTime) { + partner.bestTime = team.time; + partner.bestMapId = mapId; + } + if (day > partner.lastFinishAt) partner.lastFinishAt = day; + } + + const mapKey = `${playerId}:${partnerId}:${mapId}`; + const partnerMap = partnerMaps.get(mapKey); + if (partnerMap === undefined) { + partnerMaps.set(mapKey, { + playerId, partnerId, mapId, + bestTime: team.time, finishCount: 1, lastFinishAt: day, + }); + } else { + partnerMap.finishCount += 1; + if (team.time < partnerMap.bestTime) partnerMap.bestTime = team.time; + if (day > partnerMap.lastFinishAt) partnerMap.lastFinishAt = day; + } + } + } + } + + const partnerRows = [...partners.values()].sort( + (a, b) => a.playerId - b.playerId || a.partnerId - b.partnerId + ); + const partnerMapRows = [...partnerMaps.values()].sort( + (a, b) => a.playerId - b.playerId || a.partnerId - b.partnerId || a.mapId - b.mapId + ); + + await prisma.$transaction( + async (tx) => { + for (const rows_ of chunk(teamRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(upsertDdnetTeamRaces(JSON.stringify(rows_))); + } + for (const rows_ of chunk(rosterRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(upsertDdnetTeamRacePlayers(JSON.stringify(rows_))); + } + for (const rows_ of chunk(partnerRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(upsertDdnetPartners( + rows_.map((row) => row.playerId), + rows_.map((row) => row.partnerId), + rows_.map((row) => row.finishCount), + rows_.map((row) => row.bestTime), + rows_.map((row) => row.bestMapId), + rows_.map((row) => row.lastFinishAt) + )); + } + for (const rows_ of chunk(partnerMapRows, INSERT_CHUNK_SIZE)) { + await tx.$queryRawTyped(upsertDdnetPartnerMaps( + rows_.map((row) => row.playerId), + rows_.map((row) => row.partnerId), + rows_.map((row) => row.mapId), + rows_.map((row) => row.bestTime), + rows_.map((row) => row.finishCount), + rows_.map((row) => row.lastFinishAt) + )); + } + + await options.writeState(tx); + }, + { timeout: minutesToMilliseconds(5), maxWait: minutesToMilliseconds(1) } + ); +} diff --git a/apps/worker/src/ddnet/archive.ts b/apps/worker/src/ddnet/archive.ts new file mode 100644 index 0000000..dce0619 --- /dev/null +++ b/apps/worker/src/ddnet/archive.ts @@ -0,0 +1,39 @@ +import { GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; +import { S3_BUCKET, getS3Client } from "@teerank/teerank"; + +export function raceArchiveKey(ym: string, part: string) { + return `ddnet/race/ym=${ym}/${part}.parquet`; +} + +export function teamRaceArchiveKey(ym: string, part: string) { + return `ddnet/teamrace/ym=${ym}/${part}.parquet`; +} + +export async function putArchiveObject(key: string, body: Uint8Array) { + const s3 = getS3Client(); + + await s3.send(new PutObjectCommand({ + Bucket: S3_BUCKET, + Key: key, + Body: body, + ContentType: 'application/vnd.apache.parquet', + })); + + const head = await s3.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: key })); + + if (head.ContentLength !== body.byteLength) { + throw new Error(`Archive object ${key} size mismatch after upload`); + } +} + +export async function getArchiveObject(key: string) { + const s3 = getS3Client(); + const object = await s3.send(new GetObjectCommand({ Bucket: S3_BUCKET, Key: key })); + const body = await object.Body?.transformToByteArray(); + + if (body === undefined) { + throw new Error(`Archive object ${key} has no body`); + } + + return body; +} diff --git a/apps/worker/src/ddnet/ddnet.test.ts b/apps/worker/src/ddnet/ddnet.test.ts new file mode 100644 index 0000000..2d84fee --- /dev/null +++ b/apps/worker/src/ddnet/ddnet.test.ts @@ -0,0 +1,59 @@ +import { BoundaryTracker, monthLabel, raceRowKey } from "./state"; +import { groupTeamRows } from "./applyChunk"; +import { parseMapperNames } from "./mapsImport"; +import { ddnetMapFileName } from "./stream"; + +describe('parseMapperNames', () => { + it('splits multi-mapper conventions', () => { + expect(parseMapperNames('Ravie')).toEqual(['Ravie']); + expect(parseMapperNames('Pipou & Ravie')).toEqual(['Pipou', 'Ravie']); + expect(parseMapperNames('A, B & C')).toEqual(['A', 'B', 'C']); + expect(parseMapperNames('')).toEqual([]); + }); +}); + +describe('ddnetMapFileName', () => { + it('replaces every non-alphanumeric character with underscores', () => { + expect(ddnetMapFileName('Kobra 4')).toBe('Kobra_4'); + expect(ddnetMapFileName('#wontfix')).toBe('_wontfix'); + expect(ddnetMapFileName('Aim 10.0')).toBe('Aim_10_0'); + expect(ddnetMapFileName('-273')).toBe('_273'); + }); +}); + +describe('BoundaryTracker', () => { + it('keeps only keys within the window of the max timestamp', () => { + const tracker = new BoundaryTracker(); + tracker.add('old', new Date('2024-01-01T00:00:00Z')); + tracker.add('near', new Date('2024-01-02T23:55:00Z')); + tracker.add('max', new Date('2024-01-03T00:00:00Z')); + + const { watermark, boundary } = tracker.finalize(); + expect(watermark).toBe('2024-01-03T00:00:00.000Z'); + expect(boundary.sort()).toEqual(['max', 'near']); + }); +}); + +describe('groupTeamRows', () => { + it('groups rows by team id keeping the roster', () => { + const timestamp = new Date('2020-05-01T12:00:00Z'); + const teams = groupTeamRows([ + { teamId: 'aa', mapName: 'Kobra', playerName: 'a', time: 100, timestamp }, + { teamId: 'aa', mapName: 'Kobra', playerName: 'b', time: 100, timestamp }, + { teamId: 'bb', mapName: 'Just8', playerName: 'c', time: 50, timestamp }, + ]); + + expect(teams).toHaveLength(2); + expect(teams[0]).toMatchObject({ teamId: 'aa', playerNames: ['a', 'b'] }); + expect(teams[1]).toMatchObject({ teamId: 'bb', playerNames: ['c'] }); + }); +}); + +describe('raceRowKey / monthLabel', () => { + it('builds stable keys', () => { + const timestamp = new Date('2019-07-28T16:36:11Z'); + expect(raceRowKey({ mapName: 'Kobra', playerName: 'deen', timestamp, time: 12.34 })) + .toBe('Kobra|deen|2019-07-28T16:36:11.000Z|12.34'); + expect(monthLabel(timestamp)).toBe('2019-07'); + }); +}); diff --git a/apps/worker/src/ddnet/mapsImport.ts b/apps/worker/src/ddnet/mapsImport.ts new file mode 100644 index 0000000..d0b1a80 --- /dev/null +++ b/apps/worker/src/ddnet/mapsImport.ts @@ -0,0 +1,94 @@ +import { minutesToMilliseconds } from "date-fns"; +import { listDdnetMapsMissingThumb } from "@prisma/client/sql"; +import { getEnvInt, wait } from "@teerank/teerank"; +import { prisma } from "../prisma"; +import { DdnetMapInfoRow, DdnetMapRow, fetchMapThumbnail } from "./stream"; +import { resolveMapIds } from "./applyChunk"; + +const DDNET_THUMBNAILS_PER_RUN = getEnvInt('DDNET_THUMBNAILS_PER_RUN', 300); +const DDNET_THUMBNAIL_PAUSE_MS = getEnvInt('DDNET_THUMBNAIL_PAUSE_MS', 200); + +export function parseMapperNames(mapper: string) { + return [...new Set( + mapper.split(/,|&/).map((name) => name.trim()).filter((name) => name.length > 0) + )]; +} + +export async function importMapMetadata(mapRows: DdnetMapRow[], mapInfoRows: DdnetMapInfoRow[]) { + const mapIds = await resolveMapIds(mapRows.map((row) => row.name)); + const infoByName = new Map(mapInfoRows.map((row) => [row.name, row])); + + const previousPoints = new Map( + (await prisma.ddnetMap.findMany({ select: { mapId: true, points: true } })) + .map((row) => [row.mapId, row.points]) + ); + + const pointsChangedMapIds: number[] = []; + + await prisma.$transaction( + async (tx) => { + for (const row of mapRows) { + const mapId = mapIds.get(row.name)!; + const info = infoByName.get(row.name); + + const metadata = { + category: row.category, + points: row.points, + stars: row.stars, + mapper: row.mapper, + releasedAt: row.releasedAt, + width: info?.width ?? null, + height: info?.height ?? null, + tiles: info?.tiles ?? [], + }; + + await tx.ddnetMap.upsert({ + where: { mapId }, + create: { mapId, ...metadata }, + update: metadata, + }); + + if (previousPoints.get(mapId) !== row.points) { + pointsChangedMapIds.push(mapId); + } + + const mapperNames = parseMapperNames(row.mapper); + await tx.ddnetMapMapper.deleteMany({ where: { mapId } }); + await tx.ddnetMapMapper.createMany({ + data: mapperNames.map((mapperName) => ({ mapperName, mapId })), + skipDuplicates: true, + }); + } + }, + { timeout: minutesToMilliseconds(5), maxWait: minutesToMilliseconds(1) } + ); + + return { pointsChangedMapIds }; +} + +export async function syncMapThumbnails() { + const missing = await prisma.$queryRawTyped( + listDdnetMapsMissingThumb(DDNET_THUMBNAILS_PER_RUN) + ); + + let fetched = 0; + + for (const map of missing) { + const result = await fetchMapThumbnail(map.name, null); + + if (result.status === 'fetched') { + await prisma.ddnetMapThumb.upsert({ + where: { mapId: map.mapId }, + create: { mapId: map.mapId, png: result.png, etag: result.etag, fetchedAt: new Date() }, + update: { png: result.png, etag: result.etag, fetchedAt: new Date() }, + }); + fetched += 1; + } + + await wait(DDNET_THUMBNAIL_PAUSE_MS); + } + + if (missing.length > 0) { + console.log(`DDNet thumbnails: fetched ${fetched}/${missing.length} missing`); + } +} diff --git a/apps/worker/src/ddnet/raceParquet.ts b/apps/worker/src/ddnet/raceParquet.ts new file mode 100644 index 0000000..0d8d036 --- /dev/null +++ b/apps/worker/src/ddnet/raceParquet.ts @@ -0,0 +1,106 @@ +import { + Float32, + Float64, + Table, + TimestampMillisecond, + Utf8, + tableFromIPC, + tableToIPC, + vectorFromArray, +} from "apache-arrow"; +import { + Compression, + Table as WasmTable, + WriterPropertiesBuilder, + readParquet, + writeParquet, +} from "parquet-wasm"; +import { DdnetRaceRow, DdnetTeamRaceRow } from "./stream"; + +const SPLIT_COUNT = 25; + +function writeTable(columns: Record) { + const table = new Table(columns as never); + const wasmTable = WasmTable.fromIPCStream(tableToIPC(table, 'stream')); + const writerProperties = new WriterPropertiesBuilder() + .setCompression(Compression.ZSTD) + .build(); + + return writeParquet(wasmTable, writerProperties); +} + +export function encodeRaceRowsToParquet(rows: DdnetRaceRow[]): Uint8Array { + const columns: Record = { + mapName: vectorFromArray(rows.map((row) => row.mapName), new Utf8()), + playerName: vectorFromArray(rows.map((row) => row.playerName), new Utf8()), + time: vectorFromArray(rows.map((row) => row.time), new Float64()), + timestamp: vectorFromArray(rows.map((row) => row.timestamp.getTime()), new TimestampMillisecond()), + }; + + for (let index = 0; index < SPLIT_COUNT; index++) { + columns[`cp${index + 1}`] = vectorFromArray( + rows.map((row) => row.splits[index] ?? 0), + new Float32() + ); + } + + return writeTable(columns); +} + +export function decodeRaceRowsFromParquet(parquet: Uint8Array): DdnetRaceRow[] { + const table = tableFromIPC(readParquet(parquet).intoIPCStream()); + const rows: DdnetRaceRow[] = []; + + for (let index = 0; index < table.numRows; index++) { + const row = table.get(index)!.toJSON() as Record; + + const splits: number[] = []; + let hasData = false; + for (let cp = 1; cp <= SPLIT_COUNT; cp++) { + const value = Number(row[`cp${cp}`] ?? 0); + splits.push(value); + if (value > 0) { + hasData = true; + } + } + + rows.push({ + mapName: String(row.mapName), + playerName: String(row.playerName), + time: Number(row.time), + timestamp: new Date(Number(row.timestamp)), + splits: hasData ? splits : [], + }); + } + + return rows; +} + +export function encodeTeamRaceRowsToParquet(rows: DdnetTeamRaceRow[]): Uint8Array { + return writeTable({ + mapName: vectorFromArray(rows.map((row) => row.mapName), new Utf8()), + playerName: vectorFromArray(rows.map((row) => row.playerName), new Utf8()), + time: vectorFromArray(rows.map((row) => row.time), new Float64()), + teamId: vectorFromArray(rows.map((row) => row.teamId), new Utf8()), + timestamp: vectorFromArray(rows.map((row) => row.timestamp.getTime()), new TimestampMillisecond()), + }); +} + +export function decodeTeamRaceRowsFromParquet(parquet: Uint8Array): DdnetTeamRaceRow[] { + const table = tableFromIPC(readParquet(parquet).intoIPCStream()); + const rows: DdnetTeamRaceRow[] = []; + + for (let index = 0; index < table.numRows; index++) { + const row = table.get(index)!.toJSON() as Record; + + rows.push({ + mapName: String(row.mapName), + playerName: String(row.playerName), + time: Number(row.time), + teamId: String(row.teamId), + timestamp: new Date(Number(row.timestamp)), + }); + } + + return rows; +} diff --git a/apps/worker/src/ddnet/refresh.ts b/apps/worker/src/ddnet/refresh.ts new file mode 100644 index 0000000..3934c88 --- /dev/null +++ b/apps/worker/src/ddnet/refresh.ts @@ -0,0 +1,42 @@ +import { chunk } from "lodash"; +import { + refreshAllDdnetPoints, + refreshDdnetMapStats, + refreshDdnetPoints, + refreshDdnetPointsForMaps, + refreshDdnetPointsRanks, +} from "@prisma/client/sql"; +import { prisma } from "../prisma"; + +const REFRESH_CHUNK_SIZE = 50_000; + +export async function refreshPointsForPlayers(playerIds: number[]) { + for (const ids of chunk(playerIds, REFRESH_CHUNK_SIZE)) { + await prisma.$queryRawTyped(refreshDdnetPoints(ids)); + } +} + +export async function refreshPointsForMaps(mapIds: number[]) { + if (mapIds.length > 0) { + await prisma.$queryRawTyped(refreshDdnetPointsForMaps(mapIds)); + } +} + +export async function refreshAllPoints() { + await prisma.$queryRawTyped(refreshAllDdnetPoints()); +} + +export async function refreshPointsRanks() { + await prisma.$queryRawTyped(refreshDdnetPointsRanks()); +} + +export async function refreshMapStats(mapIds: number[]) { + for (const ids of chunk(mapIds, 500)) { + await prisma.$queryRawTyped(refreshDdnetMapStats(ids)); + } +} + +export async function refreshAllMapStats() { + const maps = await prisma.ddnetMap.findMany({ select: { mapId: true } }); + await refreshMapStats(maps.map((map) => map.mapId)); +} diff --git a/apps/worker/src/ddnet/state.ts b/apps/worker/src/ddnet/state.ts new file mode 100644 index 0000000..f09f167 --- /dev/null +++ b/apps/worker/src/ddnet/state.ts @@ -0,0 +1,77 @@ +import { Prisma } from "@prisma/client"; +import { prisma } from "../prisma"; + +export const BACKFILL_STATE_KEY = 'backfill'; +export const IMPORT_STATE_KEY = 'import'; + +// The watermark boundary set covers rows close enough to the dump cut that +// the next dump may still add neighbours with equal-or-older timestamps. +export const BOUNDARY_WINDOW_MS = 10 * 60 * 1000; + +export type ImportState = { + lastModified: string; + raceWatermark: string; + raceBoundary: string[]; + teamWatermark: string; + teamBoundary: string[]; + lastFullRefresh: string; +}; + +export async function getDdnetState(key: string): Promise { + const row = await prisma.ddnetState.findUnique({ where: { key } }); + return row === null ? null : (row.value as T); +} + +export async function setDdnetState( + tx: Prisma.TransactionClient, + key: string, + value: unknown +) { + await tx.ddnetState.upsert({ + where: { key }, + create: { key, value: value as Prisma.InputJsonValue }, + update: { value: value as Prisma.InputJsonValue }, + }); +} + +export function monthLabel(date: Date) { + return date.toISOString().slice(0, 7); +} + +export function raceRowKey(row: { mapName: string; playerName: string; timestamp: Date; time: number }) { + return `${row.mapName}|${row.playerName}|${row.timestamp.toISOString()}|${row.time}`; +} + +export class BoundaryTracker { + private entries = new Map(); + private maxMs = 0; + + add(key: string, timestamp: Date) { + const ms = timestamp.getTime(); + if (ms > this.maxMs) { + this.maxMs = ms; + if (this.entries.size > 10_000) { + this.prune(); + } + } + if (ms >= this.maxMs - BOUNDARY_WINDOW_MS) { + this.entries.set(key, ms); + } + } + + private prune() { + for (const [key, ms] of this.entries) { + if (ms < this.maxMs - BOUNDARY_WINDOW_MS) { + this.entries.delete(key); + } + } + } + + finalize() { + this.prune(); + return { + watermark: new Date(this.maxMs).toISOString(), + boundary: [...this.entries.keys()], + }; + } +} diff --git a/apps/worker/src/ddnet/stream.ts b/apps/worker/src/ddnet/stream.ts new file mode 100644 index 0000000..e9f77b9 --- /dev/null +++ b/apps/worker/src/ddnet/stream.ts @@ -0,0 +1,207 @@ +import { Readable } from "stream"; +import { parse } from "csv-parse"; +import * as unzipper from "unzipper"; +import { getEnv } from "@teerank/teerank"; + +export const DDNET_STATS_URL = getEnv('DDNET_STATS_URL', 'https://ddnet.org/stats/ddnet-stats.zip'); +export const DDNET_STATUS_BASE_URL = getEnv('DDNET_STATUS_BASE_URL', 'https://ddnet.org/status/csv'); +export const DDNET_MAP_THUMB_BASE_URL = getEnv('DDNET_MAP_THUMB_BASE_URL', 'https://ddnet.org/ranks/maps'); + +export type DdnetRaceRow = { + mapName: string; + playerName: string; + time: number; + timestamp: Date; + // 25 checkpoint times, or empty when the run recorded none. + splits: number[]; +}; + +export type DdnetTeamRaceRow = { + mapName: string; + playerName: string; + time: number; + teamId: string; // hex + timestamp: Date; +}; + +export type DdnetMapRow = { + name: string; + category: string; + points: number; + stars: number; + mapper: string; + releasedAt: Date | null; +}; + +export type DdnetMapInfoRow = { + name: string; + width: number; + height: number; + tiles: string[]; +}; + +export type DumpHandlers = { + onMap?: (row: DdnetMapRow) => void | Promise; + onMapInfo?: (row: DdnetMapInfoRow) => void | Promise; + onRace?: (row: DdnetRaceRow) => void | Promise; + onTeamRace?: (row: DdnetTeamRaceRow) => void | Promise; +}; + +// Dump timestamps have no zone marker; they are UTC on ddnet.org. +function parseDumpTimestamp(value: string): Date | null { + if (value.startsWith('0000')) { + return null; + } + + const date = new Date(value.replace(' ', 'T') + 'Z'); + return Number.isNaN(date.getTime()) ? null : date; +} + +function parseSplits(record: string[]): number[] { + const splits: number[] = []; + let hasData = false; + + for (let index = 5; index < 30; index++) { + const value = Number(record[index] ?? 0); + splits.push(value); + if (value > 0) { + hasData = true; + } + } + + return hasData ? splits : []; +} + +async function parseCsvEntry( + entry: NodeJS.ReadableStream, + onRecord: (record: string[], header: string[]) => void | Promise +) { + const parser = entry.pipe(parse({ + relax_column_count: true, + bom: true, + })); + + let header: string[] | null = null; + + for await (const record of parser as AsyncIterable) { + if (header === null) { + header = record; + continue; + } + await onRecord(record, header); + } +} + +export async function fetchDumpLastModified() { + const response = await fetch(DDNET_STATS_URL, { method: 'HEAD' }); + if (!response.ok) { + throw new Error(`HEAD ${DDNET_STATS_URL} failed: ${response.status}`); + } + return response.headers.get('last-modified') ?? ''; +} + +// Streams every CSV inside the stats dump in one pass; entries arrive in +// archive order, so handlers must not assume maps come before races. +export async function streamStatsDump(handlers: DumpHandlers) { + const response = await fetch(DDNET_STATS_URL); + + if (!response.ok || response.body === null) { + throw new Error(`GET ${DDNET_STATS_URL} failed: ${response.status}`); + } + + const zip = Readable.fromWeb(response.body as never).pipe(unzipper.Parse({ forceStream: true })); + + for await (const entry of zip as AsyncIterable) { + const fileName = entry.path.split('/').pop() ?? ''; + + if (fileName === 'maps.csv' && handlers.onMap !== undefined) { + await parseCsvEntry(entry, async (record) => { + await handlers.onMap!({ + name: record[0], + category: record[1], + points: Number(record[2]), + stars: Number(record[3]), + mapper: record[4], + releasedAt: parseDumpTimestamp(record[5]), + }); + }); + } else if (fileName === 'mapinfo.csv' && handlers.onMapInfo !== undefined) { + await parseCsvEntry(entry, async (record, header) => { + const tiles: string[] = []; + for (let index = 3; index < header.length; index++) { + if (Number(record[index]) > 0) { + tiles.push(header[index]); + } + } + await handlers.onMapInfo!({ + name: record[0], + width: Number(record[1]), + height: Number(record[2]), + tiles, + }); + }); + } else if (fileName === 'race.csv' && handlers.onRace !== undefined) { + await parseCsvEntry(entry, async (record) => { + const timestamp = parseDumpTimestamp(record[3]); + if (timestamp === null) { + return; + } + await handlers.onRace!({ + mapName: record[0], + playerName: record[1], + time: Number(record[2]), + timestamp, + splits: parseSplits(record), + }); + }); + } else if (fileName === 'teamrace.csv' && handlers.onTeamRace !== undefined) { + await parseCsvEntry(entry, async (record) => { + const timestamp = parseDumpTimestamp(record[4]); + if (timestamp === null) { + return; + } + await handlers.onTeamRace!({ + mapName: record[0], + playerName: record[1], + time: Number(record[2]), + teamId: record[3].toLowerCase(), + timestamp, + }); + }); + } else { + entry.autodrain(); + } + } +} + +// Sanitisation rule used by ddnet.org for map asset file names. +export function ddnetMapFileName(mapName: string) { + return mapName.replace(/[^A-Za-z0-9]/g, '_'); +} + +export async function fetchMapThumbnail(mapName: string, etag: string | null) { + const url = `${DDNET_MAP_THUMB_BASE_URL}/${encodeURIComponent(ddnetMapFileName(mapName))}.png`; + const headers: Record = {}; + + if (etag !== null) { + headers['If-None-Match'] = etag; + } + + const response = await fetch(url, { headers }); + + if (response.status === 304) { + return { status: 'unchanged' as const }; + } + if (response.status === 404) { + return { status: 'missing' as const }; + } + if (!response.ok) { + throw new Error(`GET ${url} failed: ${response.status}`); + } + + return { + status: 'fetched' as const, + png: Buffer.from(await response.arrayBuffer()), + etag: response.headers.get('etag'), + }; +} diff --git a/apps/worker/src/main.ts b/apps/worker/src/main.ts index af03d89..a9be1d2 100644 --- a/apps/worker/src/main.ts +++ b/apps/worker/src/main.ts @@ -9,6 +9,9 @@ import { startUpdateGlobalCountsWorker } from "./workers/updateGlobalCounts"; import { startArchiveSnapshotsWorker } from "./workers/archiveSnapshots"; import { startRollupDayWorker } from "./workers/rollupDay"; import { startRollupBackfillWorker } from "./workers/rollupBackfill"; +import { startDdnetBackfillWorker } from "./workers/ddnetBackfill"; +import { startDdnetImportWorker } from "./workers/ddnetImport"; +import { startDdnetOnlineWorker } from "./workers/ddnetOnlineImport"; async function main() { const workers = await Promise.all([ @@ -23,6 +26,9 @@ async function main() { startArchiveSnapshotsWorker(), startRollupDayWorker(), startRollupBackfillWorker(), + startDdnetBackfillWorker(), + startDdnetImportWorker(), + startDdnetOnlineWorker(), ]); async function gracefulShutdown(signal: string) { diff --git a/apps/worker/src/workers/ddnetBackfill.ts b/apps/worker/src/workers/ddnetBackfill.ts new file mode 100644 index 0000000..7b1b8f4 --- /dev/null +++ b/apps/worker/src/workers/ddnetBackfill.ts @@ -0,0 +1,326 @@ +import { getEnvInt, processDdnetBackfillJobs, wait } from "@teerank/teerank"; +import { prisma } from "../prisma"; +import { + DdnetMapInfoRow, + DdnetMapRow, + DdnetRaceRow, + DdnetTeamRaceRow, + fetchDumpLastModified, + streamStatsDump, +} from "../ddnet/stream"; +import { + decodeRaceRowsFromParquet, + decodeTeamRaceRowsFromParquet, + encodeRaceRowsToParquet, + encodeTeamRaceRowsToParquet, +} from "../ddnet/raceParquet"; +import { putArchiveObject, getArchiveObject, raceArchiveKey, teamRaceArchiveKey } from "../ddnet/archive"; +import { + RecordCandidate, + applyRaceRows, + applyTeamRuns, + ensureFinishPartitions, + groupTeamRows, + loadRecordState, + replayRecordCandidates, +} from "../ddnet/applyChunk"; +import { importMapMetadata } from "../ddnet/mapsImport"; +import { refreshAllPoints, refreshAllMapStats, refreshPointsRanks } from "../ddnet/refresh"; +import { + BACKFILL_STATE_KEY, + BoundaryTracker, + IMPORT_STATE_KEY, + ImportState, + getDdnetState, + monthLabel, + raceRowKey, + setDdnetState, +} from "../ddnet/state"; + +const DDNET_TIME_BUDGET_MS = getEnvInt('DDNET_TIME_BUDGET_MS', 10 * 60 * 1000); +const DDNET_BATCH_PAUSE_MS = getEnvInt('DDNET_BATCH_PAUSE_MS', 200); +const DDNET_BUFFER_ROWS = getEnvInt('DDNET_BUFFER_ROWS', 200_000); + +export type BackfillState = { + phase: 'download' | 'apply' | 'teams' | 'tail' | 'done'; + lastModified?: string; + raceMonths?: string[]; + raceParts?: Record; + teamMonths?: string[]; + teamParts?: Record; + monthIndex?: number; + part?: number; + candidates?: RecordCandidate[]; + raceWatermark?: string; + teamWatermark?: string; + raceBoundary?: string[]; + teamBoundary?: string[]; +}; + +// Buffers rows per month and flushes the largest month once the global cap +// is hit, so the unordered 69M-row stream never holds more than the cap. +class MonthBuffers { + private buffers = new Map(); + private partCounts: Record = {}; + private total = 0; + + constructor( + private encode: (rows: Row[]) => Uint8Array, + private keyFor: (ym: string, part: string) => string + ) {} + + async add(ym: string, row: Row) { + let buffer = this.buffers.get(ym); + if (buffer === undefined) { + buffer = []; + this.buffers.set(ym, buffer); + this.partCounts[ym] ??= 0; + } + buffer.push(row); + this.total += 1; + + if (this.total >= DDNET_BUFFER_ROWS) { + let largest: string | null = null; + for (const [key, rows] of this.buffers) { + if (largest === null || rows.length > this.buffers.get(largest)!.length) { + largest = key; + } + } + await this.flush(largest!); + } + } + + private async flush(ym: string) { + const rows = this.buffers.get(ym); + if (rows === undefined || rows.length === 0) { + return; + } + + const part = this.partCounts[ym]; + await putArchiveObject(this.keyFor(ym, `part-${part}`), this.encode(rows)); + this.partCounts[ym] = part + 1; + this.total -= rows.length; + this.buffers.set(ym, []); + } + + async flushAll() { + for (const ym of this.buffers.keys()) { + await this.flush(ym); + } + return this.partCounts; + } +} + +async function downloadPhase() { + const lastModified = await fetchDumpLastModified(); + + const raceBuffers = new MonthBuffers(encodeRaceRowsToParquet, raceArchiveKey); + const teamBuffers = new MonthBuffers( + (teams) => encodeTeamRaceRowsToParquet(teams.flat()), + teamRaceArchiveKey + ); + + const mapRows: DdnetMapRow[] = []; + const mapInfoRows: DdnetMapInfoRow[] = []; + const raceBoundary = new BoundaryTracker(); + const teamBoundary = new BoundaryTracker(); + + // A team's rows are contiguous in the CSV and share one timestamp; buffer + // them as one unit so a flush can never split a team across parts. + let pendingTeam: DdnetTeamRaceRow[] = []; + let raceCount = 0; + + const flushPendingTeam = async () => { + if (pendingTeam.length > 0) { + await teamBuffers.add(monthLabel(pendingTeam[0].timestamp), pendingTeam); + teamBoundary.add(pendingTeam[0].teamId, pendingTeam[0].timestamp); + pendingTeam = []; + } + }; + + await streamStatsDump({ + onMap: (row) => { + mapRows.push(row); + }, + onMapInfo: (row) => { + mapInfoRows.push(row); + }, + onRace: async (row) => { + await raceBuffers.add(monthLabel(row.timestamp), row); + raceBoundary.add(raceRowKey(row), row.timestamp); + raceCount += 1; + if (raceCount % 5_000_000 === 0) { + console.log(`DDNet backfill download: ${raceCount} race rows streamed`); + } + }, + onTeamRace: async (row) => { + if (pendingTeam.length > 0 && pendingTeam[0].teamId !== row.teamId) { + await flushPendingTeam(); + } + pendingTeam.push(row); + }, + }); + await flushPendingTeam(); + + const raceParts = await raceBuffers.flushAll(); + const teamParts = await teamBuffers.flushAll(); + + await importMapMetadata(mapRows, mapInfoRows); + + const race = raceBoundary.finalize(); + const team = teamBoundary.finalize(); + + const state: BackfillState = { + phase: 'apply', + lastModified, + raceMonths: Object.keys(raceParts).sort(), + raceParts, + teamMonths: Object.keys(teamParts).sort(), + teamParts, + monthIndex: 0, + part: -1, + candidates: [], + raceWatermark: race.watermark, + raceBoundary: race.boundary, + teamWatermark: team.watermark, + teamBoundary: team.boundary, + }; + + await prisma.$transaction((tx) => setDdnetState(tx, BACKFILL_STATE_KEY, state)); + console.log( + `DDNet backfill download done: ${raceCount} race rows over ` + + `${state.raceMonths!.length} months, dump ${lastModified}` + ); +} + +async function applyPhase(state: BackfillState, startedAt: number) { + const recordState = await loadRecordState(); + + while (Date.now() - startedAt < DDNET_TIME_BUDGET_MS) { + const monthIndex = state.monthIndex!; + const month = state.raceMonths![monthIndex]; + + if (month === undefined) { + state = { ...state, phase: 'teams', monthIndex: 0, part: -1 }; + await prisma.$transaction((tx) => setDdnetState(tx, BACKFILL_STATE_KEY, state)); + return state; + } + + const partCount = state.raceParts![month]; + const part = state.part! + 1; + + if (part >= partCount) { + // Month complete: replay its record candidates chronologically. + const candidates = state.candidates ?? []; + const next: BackfillState = { ...state, monthIndex: monthIndex + 1, part: -1, candidates: [] }; + await prisma.$transaction(async (tx) => { + const events = await replayRecordCandidates(candidates, recordState, tx); + await setDdnetState(tx, BACKFILL_STATE_KEY, next); + console.log(`DDNet backfill: month ${month} done, ${events} record events`); + }); + state = next; + continue; + } + + await ensureFinishPartitions(month); + + const rows = decodeRaceRowsFromParquet(await getArchiveObject(raceArchiveKey(month, `part-${part}`))); + + await applyRaceRows(rows, recordState, { + dayCounts: 'increment', + writeState: async (tx, candidates) => { + state = { ...state, part, candidates: [...(state.candidates ?? []), ...candidates] }; + await setDdnetState(tx, BACKFILL_STATE_KEY, state); + }, + }); + + console.log(`DDNet backfill: applied race ${month} part ${part + 1}/${partCount} (${rows.length} rows)`); + await wait(DDNET_BATCH_PAUSE_MS); + } + + return state; +} + +async function teamsPhase(state: BackfillState, startedAt: number) { + while (Date.now() - startedAt < DDNET_TIME_BUDGET_MS) { + const monthIndex = state.monthIndex!; + const month = state.teamMonths![monthIndex]; + + if (month === undefined) { + state = { ...state, phase: 'tail' }; + await prisma.$transaction((tx) => setDdnetState(tx, BACKFILL_STATE_KEY, state)); + return state; + } + + const partCount = state.teamParts![month]; + const part = state.part! + 1; + + if (part >= partCount) { + state = { ...state, monthIndex: monthIndex + 1, part: -1 }; + await prisma.$transaction((tx) => setDdnetState(tx, BACKFILL_STATE_KEY, state)); + continue; + } + + const rows = decodeTeamRaceRowsFromParquet( + await getArchiveObject(teamRaceArchiveKey(month, `part-${part}`)) + ); + const teams = groupTeamRows(rows); + + await applyTeamRuns(teams, { + writeState: async (tx) => { + state = { ...state, part }; + await setDdnetState(tx, BACKFILL_STATE_KEY, state); + }, + }); + + console.log(`DDNet backfill: applied teamrace ${month} part ${part + 1}/${partCount} (${teams.length} teams)`); + await wait(DDNET_BATCH_PAUSE_MS); + } + + return state; +} + +async function tailPhase(state: BackfillState) { + console.log('DDNet backfill: tail refresh starting'); + await refreshAllPoints(); + await refreshPointsRanks(); + await refreshAllMapStats(); + + const importState: ImportState = { + lastModified: state.lastModified!, + raceWatermark: state.raceWatermark!, + raceBoundary: state.raceBoundary ?? [], + teamWatermark: state.teamWatermark!, + teamBoundary: state.teamBoundary ?? [], + lastFullRefresh: new Date().toISOString(), + }; + + await prisma.$transaction(async (tx) => { + await setDdnetState(tx, IMPORT_STATE_KEY, importState); + await setDdnetState(tx, BACKFILL_STATE_KEY, { phase: 'done' } satisfies BackfillState); + }); + console.log('DDNet backfill: done'); +} + +export async function ddnetBackfill() { + const startedAt = Date.now(); + let state = await getDdnetState(BACKFILL_STATE_KEY) ?? { phase: 'download' as const }; + + if (state.phase === 'download') { + await downloadPhase(); + return; + } + if (state.phase === 'apply') { + state = await applyPhase(state, startedAt); + } + if (state.phase === 'teams') { + state = await teamsPhase(state, startedAt); + } + if (state.phase === 'tail') { + await tailPhase(state); + } +} + +export async function startDdnetBackfillWorker() { + return processDdnetBackfillJobs(ddnetBackfill); +} diff --git a/apps/worker/src/workers/ddnetImport.ts b/apps/worker/src/workers/ddnetImport.ts new file mode 100644 index 0000000..6966ae5 --- /dev/null +++ b/apps/worker/src/workers/ddnetImport.ts @@ -0,0 +1,225 @@ +import { differenceInDays } from "date-fns"; +import { processDdnetImportJobs } from "@teerank/teerank"; +import { + DdnetMapInfoRow, + DdnetMapRow, + DdnetRaceRow, + DdnetTeamRaceRow, + fetchDumpLastModified, + streamStatsDump, +} from "../ddnet/stream"; +import { encodeRaceRowsToParquet, encodeTeamRaceRowsToParquet } from "../ddnet/raceParquet"; +import { putArchiveObject, raceArchiveKey, teamRaceArchiveKey } from "../ddnet/archive"; +import { + applyRaceRows, + applyTeamRuns, + ensureFinishPartitions, + groupTeamRows, + loadRecordState, + replayRecordCandidates, +} from "../ddnet/applyChunk"; +import { importMapMetadata, syncMapThumbnails } from "../ddnet/mapsImport"; +import { + refreshAllPoints, + refreshAllMapStats, + refreshMapStats, + refreshPointsForMaps, + refreshPointsForPlayers, + refreshPointsRanks, +} from "../ddnet/refresh"; +import { + BACKFILL_STATE_KEY, + BOUNDARY_WINDOW_MS, + BoundaryTracker, + IMPORT_STATE_KEY, + ImportState, + getDdnetState, + monthLabel, + raceRowKey, + setDdnetState, +} from "../ddnet/state"; +import { prisma } from "../prisma"; + +// Whole trailing days are re-counted on every run, which both covers late +// rows near the dump cut and heals finish counts wiped by a re-rollup. +const RECOUNT_DAYS = 3; + +function startOfRecountWindow(watermark: Date) { + const day = new Date(Date.UTC( + watermark.getUTCFullYear(), watermark.getUTCMonth(), watermark.getUTCDate() + )); + day.setUTCDate(day.getUTCDate() - (RECOUNT_DAYS - 1)); + return day; +} + +async function archiveNewRows( + raceRows: DdnetRaceRow[], + teamRows: DdnetTeamRaceRow[], + dumpLabel: string +) { + const raceByMonth = new Map(); + for (const row of raceRows) { + const ym = monthLabel(row.timestamp); + (raceByMonth.get(ym) ?? raceByMonth.set(ym, []).get(ym)!).push(row); + } + for (const [ym, rows] of raceByMonth) { + await putArchiveObject(raceArchiveKey(ym, `dump-${dumpLabel}`), encodeRaceRowsToParquet(rows)); + } + + const teamByMonth = new Map(); + for (const row of teamRows) { + const ym = monthLabel(row.timestamp); + (teamByMonth.get(ym) ?? teamByMonth.set(ym, []).get(ym)!).push(row); + } + for (const [ym, rows] of teamByMonth) { + await putArchiveObject(teamRaceArchiveKey(ym, `dump-${dumpLabel}`), encodeTeamRaceRowsToParquet(rows)); + } +} + +export async function ddnetImport() { + const backfill = await getDdnetState<{ phase: string }>(BACKFILL_STATE_KEY); + if (backfill?.phase !== 'done') { + return; + } + + const state = await getDdnetState(IMPORT_STATE_KEY); + if (state === null) { + throw new Error('DDNet import state missing while backfill is done'); + } + + const lastModified = await fetchDumpLastModified(); + if (lastModified === state.lastModified) { + await syncMapThumbnails(); + return; + } + + const raceWatermark = new Date(state.raceWatermark); + const teamWatermark = new Date(state.teamWatermark); + const windowStart = startOfRecountWindow(raceWatermark); + const raceBoundary = new Set(state.raceBoundary); + const teamBoundary = new Set(state.teamBoundary); + + const mapRows: DdnetMapRow[] = []; + const mapInfoRows: DdnetMapInfoRow[] = []; + const windowRows: DdnetRaceRow[] = []; + const teamWindowRows: DdnetTeamRaceRow[] = []; + + const raceTracker = new BoundaryTracker(); + const teamTracker = new BoundaryTracker(); + + await streamStatsDump({ + onMap: (row) => { + mapRows.push(row); + }, + onMapInfo: (row) => { + mapInfoRows.push(row); + }, + onRace: (row) => { + if (row.timestamp >= windowStart) { + windowRows.push(row); + raceTracker.add(raceRowKey(row), row.timestamp); + } + }, + onTeamRace: (row) => { + if (row.timestamp.getTime() > teamWatermark.getTime() - BOUNDARY_WINDOW_MS) { + teamWindowRows.push(row); + } + }, + }); + + const isNewRace = (row: DdnetRaceRow) => { + const ms = row.timestamp.getTime(); + if (ms > raceWatermark.getTime()) { + return true; + } + return ms > raceWatermark.getTime() - BOUNDARY_WINDOW_MS && !raceBoundary.has(raceRowKey(row)); + }; + + const newRaceRows = windowRows.filter(isNewRace); + + const teams = groupTeamRows(teamWindowRows); + const newTeams = teams.filter((team) => { + const ms = team.timestamp.getTime(); + if (ms > teamWatermark.getTime()) { + return true; + } + return !teamBoundary.has(team.teamId); + }); + for (const team of teams) { + teamTracker.add(team.teamId, team.timestamp); + } + + const dumpLabel = new Date(lastModified).toISOString().slice(0, 10); + await archiveNewRows(newRaceRows, teamWindowRows.filter((row) => + newTeams.some((team) => team.teamId === row.teamId) + ), dumpLabel); + + const { pointsChangedMapIds } = await importMapMetadata(mapRows, mapInfoRows); + + for (const ym of new Set(windowRows.map((row) => monthLabel(row.timestamp)))) { + await ensureFinishPartitions(ym); + } + + const recordState = await loadRecordState(); + const race = raceTracker.finalize(); + + const applied = await applyRaceRows(windowRows, recordState, { + dayCounts: 'set', + newRows: isNewRace, + writeState: async (tx, candidates) => { + await replayRecordCandidates(candidates, recordState, tx); + await setDdnetState(tx, IMPORT_STATE_KEY, { + ...state, + raceWatermark: race.watermark, + raceBoundary: race.boundary, + } satisfies ImportState); + }, + }); + + const team = teamTracker.finalize(); + + await applyTeamRuns(newTeams, { + writeState: async (tx) => { + await setDdnetState(tx, IMPORT_STATE_KEY, { + ...state, + raceWatermark: race.watermark, + raceBoundary: race.boundary, + teamWatermark: team.watermark, + teamBoundary: team.boundary, + lastModified, + } satisfies ImportState); + }, + }); + + await refreshPointsForPlayers(applied.playerIds); + await refreshPointsForMaps(pointsChangedMapIds); + await refreshMapStats(applied.mapIds); + await refreshPointsRanks(); + + if (differenceInDays(new Date(), new Date(state.lastFullRefresh)) >= 7) { + console.log('DDNet import: weekly full refresh'); + await refreshAllPoints(); + await refreshPointsRanks(); + await refreshAllMapStats(); + await prisma.$transaction((tx) => setDdnetState(tx, IMPORT_STATE_KEY, { + ...state, + raceWatermark: race.watermark, + raceBoundary: race.boundary, + teamWatermark: team.watermark, + teamBoundary: team.boundary, + lastModified, + lastFullRefresh: new Date().toISOString(), + } satisfies ImportState)); + } + + await syncMapThumbnails(); + + console.log( + `DDNet import: dump ${dumpLabel} applied — ${newRaceRows.length} new finishes, ` + + `${newTeams.length} new team runs, ${pointsChangedMapIds.length} maps re-pointed` + ); +} + +export async function startDdnetImportWorker() { + return processDdnetImportJobs(ddnetImport); +} diff --git a/apps/worker/src/workers/ddnetOnlineImport.ts b/apps/worker/src/workers/ddnetOnlineImport.ts new file mode 100644 index 0000000..0ce98b4 --- /dev/null +++ b/apps/worker/src/workers/ddnetOnlineImport.ts @@ -0,0 +1,185 @@ +import { minutesToMilliseconds } from "date-fns"; +import { processDdnetOnlineJobs } from "@teerank/teerank"; +import { prisma } from "../prisma"; +import { DDNET_STATUS_BASE_URL } from "../ddnet/stream"; +import { getDdnetState, setDdnetState } from "../ddnet/state"; + +const SOURCES = [ + { stateKey: 'online-bycountry', kind: 0, url: `${DDNET_STATUS_BASE_URL}/bycountry`, withTotal: true }, + { stateKey: 'online-bymod', kind: 1, url: `${DDNET_STATUS_BASE_URL}/bymod`, withTotal: false }, +]; + +const FLUSH_DAY_COUNT = 60; + +type OnlineState = { offset: number }; + +type DayAggregate = { + day: string; + samples: number; + byKey: Map; + totalSum: number; + totalMax: number; +}; + +function newDayAggregate(day: string): DayAggregate { + return { day, samples: 0, byKey: new Map(), totalSum: 0, totalMax: 0 }; +} + +function finalizeDay(aggregate: DayAggregate, kind: number, withTotal: boolean) { + const day = new Date(`${aggregate.day}T00:00:00Z`); + const rows: { day: Date; kind: number; key: string; avgPlayers: number; maxPlayers: number }[] = []; + + for (const [key, value] of aggregate.byKey) { + rows.push({ + day, + kind, + key, + // A key missing from a sample counts as zero, so the divisor is the + // day's full sample count. + avgPlayers: Math.round(value.sum / aggregate.samples), + maxPlayers: value.max, + }); + } + + if (withTotal) { + rows.push({ + day, + kind, + key: 'ALL', + avgPlayers: Math.round(aggregate.totalSum / aggregate.samples), + maxPlayers: aggregate.totalMax, + }); + } + + return rows; +} + +async function flushDays( + rows: { day: Date; kind: number; key: string; avgPlayers: number; maxPlayers: number }[], + kind: number, + stateKey: string, + offset: number +) { + const days = [...new Set(rows.map((row) => row.day.getTime()))].map((ms) => new Date(ms)); + + await prisma.$transaction( + async (tx) => { + await tx.ddnetOnlineDay.deleteMany({ where: { kind, day: { in: days } } }); + await tx.ddnetOnlineDay.createMany({ data: rows }); + await setDdnetState(tx, stateKey, { offset } satisfies OnlineState); + }, + { timeout: minutesToMilliseconds(5), maxWait: minutesToMilliseconds(1) } + ); +} + +async function importSource(source: typeof SOURCES[number]) { + const state = await getDdnetState(source.stateKey); + let offset = state?.offset ?? 0; + + const response = await fetch(source.url, { + headers: offset > 0 ? { Range: `bytes=${offset}-` } : {}, + }); + + if (response.status === 416) { + return; // Nothing new past the offset. + } + if (!response.ok || response.body === null) { + throw new Error(`GET ${source.url} failed: ${response.status}`); + } + if (response.status === 200) { + offset = 0; // Server ignored the range or the file was rewritten. + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + let pendingRows: ReturnType = []; + let current: DayAggregate | null = null; + let currentDayStart = offset; + let lineStart = offset; + let buffered = ''; + + const handleLine = (line: string) => { + // "YYYY-MM-DD HH:MM,KEY:count,KEY:count,..." + const day = line.slice(0, 10); + if (!/^\d{4}-\d{2}-\d{2}$/.test(day)) { + return; + } + + if (current === null || current.day !== day) { + if (current !== null) { + pendingRows.push(...finalizeDay(current, source.kind, source.withTotal)); + } + current = newDayAggregate(day); + currentDayStart = lineStart; + } + + current.samples += 1; + let total = 0; + + const fields = line.split(','); + for (let index = 1; index < fields.length; index++) { + const separator = fields[index].lastIndexOf(':'); + if (separator === -1) { + continue; + } + const key = fields[index].slice(0, separator); + const count = Number(fields[index].slice(separator + 1)); + if (!Number.isFinite(count)) { + continue; + } + + total += count; + const entry = current.byKey.get(key); + if (entry === undefined) { + current.byKey.set(key, { sum: count, max: count }); + } else { + entry.sum += count; + if (count > entry.max) entry.max = count; + } + } + + if (total > current.totalMax) current.totalMax = total; + current.totalSum += total; + }; + + for (;;) { + const { done, value } = await reader.read(); + const text = done ? decoder.decode() : decoder.decode(value, { stream: true }); + buffered += text; + + let newlineIndex: number; + while ((newlineIndex = buffered.indexOf('\n')) !== -1) { + const line = buffered.slice(0, newlineIndex); + handleLine(line.trimEnd()); + lineStart += Buffer.byteLength(line, 'utf8') + 1; + buffered = buffered.slice(newlineIndex + 1); + } + + if (pendingRows.length >= FLUSH_DAY_COUNT * 10) { + // The offset stays at the start of the still-open day so the next run + // re-reads and re-finalizes it. + await flushDays(pendingRows, source.kind, source.stateKey, currentDayStart); + pendingRows = []; + } + + if (done) { + break; + } + } + + // The last (incomplete) day is intentionally not finalized. + if (pendingRows.length > 0) { + await flushDays(pendingRows, source.kind, source.stateKey, currentDayStart); + } +} + +export async function ddnetOnlineImport() { + for (const source of SOURCES) { + await importSource(source); + } +} + +export async function startDdnetOnlineWorker() { + return processDdnetOnlineJobs(ddnetOnlineImport); +} diff --git a/libs/prisma/prisma/migrations/20260826000000_ddnet_tables/migration.sql b/libs/prisma/prisma/migrations/20260826000000_ddnet_tables/migration.sql new file mode 100644 index 0000000..270b990 --- /dev/null +++ b/libs/prisma/prisma/migrations/20260826000000_ddnet_tables/migration.sql @@ -0,0 +1,165 @@ +-- DDNet official stats tables. Same conventions as the rollup tables: no +-- foreign keys, ids resolved at write time. finishCount on the partitioned +-- rollup tables is added with a constant default, which is metadata-only. + +-- AlterTable +ALTER TABLE "PlayerDay" ADD COLUMN "finishCount" INTEGER NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "MapDay" ADD COLUMN "finishCount" INTEGER NOT NULL DEFAULT 0; + +-- CreateTable +CREATE TABLE "DdnetMap" ( + "mapId" INTEGER NOT NULL, + "category" TEXT NOT NULL, + "points" SMALLINT NOT NULL, + "stars" SMALLINT NOT NULL, + "mapper" TEXT NOT NULL, + "releasedAt" DATE, + "width" INTEGER, + "height" INTEGER, + "tiles" TEXT[], + "finishCount" INTEGER NOT NULL DEFAULT 0, + "medianTime" DOUBLE PRECISION, + "wrSplits" REAL[], + "medianSplits" REAL[], + "timeQuantiles" REAL[], + + CONSTRAINT "DdnetMap_pkey" PRIMARY KEY ("mapId") +); + +-- CreateTable +CREATE TABLE "DdnetMapMapper" ( + "mapperName" TEXT NOT NULL, + "mapId" INTEGER NOT NULL, + + CONSTRAINT "DdnetMapMapper_pkey" PRIMARY KEY ("mapperName","mapId") +); + +-- CreateIndex +CREATE INDEX "DdnetMapMapper_mapId_idx" ON "DdnetMapMapper"("mapId"); + +-- CreateTable +CREATE TABLE "DdnetMapThumb" ( + "mapId" INTEGER NOT NULL, + "png" BYTEA NOT NULL, + "etag" TEXT, + "fetchedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "DdnetMapThumb_pkey" PRIMARY KEY ("mapId") +); + +-- CreateTable +CREATE TABLE "DdnetRaceBest" ( + "mapId" INTEGER NOT NULL, + "playerId" INTEGER NOT NULL, + "time" DOUBLE PRECISION NOT NULL, + "finishCount" INTEGER NOT NULL, + "firstFinishAt" DATE NOT NULL, + "bestAt" DATE NOT NULL, + "splits" REAL[], + + CONSTRAINT "DdnetRaceBest_pkey" PRIMARY KEY ("mapId","playerId") +); + +-- CreateIndex +CREATE INDEX "DdnetRaceBest_mapId_time_idx" ON "DdnetRaceBest"("mapId", "time"); + +-- CreateIndex +CREATE INDEX "DdnetRaceBest_playerId_idx" ON "DdnetRaceBest"("playerId"); + +-- CreateTable +CREATE TABLE "DdnetPlayer" ( + "playerId" INTEGER NOT NULL, + "points" INTEGER NOT NULL DEFAULT 0, + "pointsRank" INTEGER, + "finishCount" INTEGER NOT NULL DEFAULT 0, + "firstFinishAt" TIMESTAMP(3) NOT NULL, + "lastFinishAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "DdnetPlayer_pkey" PRIMARY KEY ("playerId") +); + +-- CreateIndex +CREATE INDEX "DdnetPlayer_points_idx" ON "DdnetPlayer"("points" DESC); + +-- CreateTable +CREATE TABLE "DdnetTeamRace" ( + "teamId" BYTEA NOT NULL, + "mapId" INTEGER NOT NULL, + "time" DOUBLE PRECISION NOT NULL, + "timestamp" TIMESTAMP(3) NOT NULL, + "size" SMALLINT NOT NULL, + + CONSTRAINT "DdnetTeamRace_pkey" PRIMARY KEY ("teamId") +); + +-- CreateIndex +CREATE INDEX "DdnetTeamRace_mapId_time_idx" ON "DdnetTeamRace"("mapId", "time"); + +-- CreateTable +CREATE TABLE "DdnetTeamRacePlayer" ( + "teamId" BYTEA NOT NULL, + "playerId" INTEGER NOT NULL, + + CONSTRAINT "DdnetTeamRacePlayer_pkey" PRIMARY KEY ("teamId","playerId") +); + +-- CreateIndex +CREATE INDEX "DdnetTeamRacePlayer_playerId_idx" ON "DdnetTeamRacePlayer"("playerId"); + +-- CreateTable +CREATE TABLE "DdnetPartner" ( + "playerId" INTEGER NOT NULL, + "partnerId" INTEGER NOT NULL, + "finishCount" INTEGER NOT NULL, + "bestTime" DOUBLE PRECISION NOT NULL, + "bestMapId" INTEGER NOT NULL, + "lastFinishAt" DATE NOT NULL, + + CONSTRAINT "DdnetPartner_pkey" PRIMARY KEY ("playerId","partnerId") +); + +-- CreateTable +CREATE TABLE "DdnetPartnerMap" ( + "playerId" INTEGER NOT NULL, + "partnerId" INTEGER NOT NULL, + "mapId" INTEGER NOT NULL, + "bestTime" DOUBLE PRECISION NOT NULL, + "finishCount" INTEGER NOT NULL, + "lastFinishAt" DATE NOT NULL, + + CONSTRAINT "DdnetPartnerMap_pkey" PRIMARY KEY ("playerId","partnerId","mapId") +); + +-- CreateTable +CREATE TABLE "DdnetRecord" ( + "mapId" INTEGER NOT NULL, + "timestamp" TIMESTAMP(3) NOT NULL, + "playerId" INTEGER NOT NULL, + "time" DOUBLE PRECISION NOT NULL, + + CONSTRAINT "DdnetRecord_pkey" PRIMARY KEY ("mapId","timestamp","playerId") +); + +-- CreateIndex +CREATE INDEX "DdnetRecord_timestamp_idx" ON "DdnetRecord"("timestamp"); + +-- CreateTable +CREATE TABLE "DdnetOnlineDay" ( + "day" DATE NOT NULL, + "kind" SMALLINT NOT NULL, + "key" TEXT NOT NULL, + "avgPlayers" INTEGER NOT NULL, + "maxPlayers" INTEGER NOT NULL, + + CONSTRAINT "DdnetOnlineDay_pkey" PRIMARY KEY ("day","kind","key") +); + +-- CreateTable +CREATE TABLE "DdnetState" ( + "key" TEXT NOT NULL, + "value" JSONB NOT NULL, + + CONSTRAINT "DdnetState_pkey" PRIMARY KEY ("key") +); diff --git a/libs/prisma/prisma/schema.prisma b/libs/prisma/prisma/schema.prisma index 60b0f4f..cfd6ab3 100644 --- a/libs/prisma/prisma/schema.prisma +++ b/libs/prisma/prisma/schema.prisma @@ -335,9 +335,10 @@ model GlobalDay { } model PlayerDay { - day DateTime @db.Date - playerId Int - playTime Int // seconds + day DateTime @db.Date + playerId Int + playTime Int // seconds + finishCount Int @default(0) @@id([day, playerId]) @@index([playerId, day]) @@ -357,6 +358,7 @@ model MapDay { mapId Int playTime Int playerCount Int + finishCount Int @default(0) @@id([mapId, day]) } @@ -387,3 +389,125 @@ model PlayerPartner { @@id([playerId, partnerId]) } + +model DdnetMap { + mapId Int @id // Map.id, gameTypeName 'DDraceNetwork' + category String // Novice / Moderate / Brutal / Insane / Solo / ... + points Int @db.SmallInt + stars Int @db.SmallInt + mapper String // raw, possibly "A & B"; parsed into DdnetMapMapper + releasedAt DateTime? @db.Date + width Int? + height Int? + tiles String[] // names of the set tile-feature flags + finishCount Int @default(0) + medianTime Float? + wrSplits Float[] @db.Real + medianSplits Float[] @db.Real + timeQuantiles Float[] @db.Real // 101 points, 0..100th percentile +} + +model DdnetMapMapper { + mapperName String + mapId Int + + @@id([mapperName, mapId]) + @@index([mapId]) +} + +model DdnetMapThumb { + mapId Int @id + png Bytes + etag String? + fetchedAt DateTime +} + +model DdnetRaceBest { + mapId Int + playerId Int + time Float + finishCount Int + firstFinishAt DateTime @db.Date + bestAt DateTime @db.Date + splits Float[] @db.Real // empty when the best run has no checkpoints + + @@id([mapId, playerId]) + @@index([mapId, time]) + @@index([playerId]) +} + +model DdnetPlayer { + playerId Int @id + points Int @default(0) + pointsRank Int? + finishCount Int @default(0) + firstFinishAt DateTime + lastFinishAt DateTime + + @@index([points(sort: Desc)]) +} + +model DdnetTeamRace { + teamId Bytes @id + mapId Int + time Float + timestamp DateTime + size Int @db.SmallInt + + @@index([mapId, time]) +} + +model DdnetTeamRacePlayer { + teamId Bytes + playerId Int + + @@id([teamId, playerId]) + @@index([playerId]) +} + +model DdnetPartner { + playerId Int + partnerId Int + finishCount Int + bestTime Float + bestMapId Int + lastFinishAt DateTime @db.Date + + @@id([playerId, partnerId]) +} + +model DdnetPartnerMap { + playerId Int + partnerId Int + mapId Int + bestTime Float + finishCount Int + lastFinishAt DateTime @db.Date + + @@id([playerId, partnerId, mapId]) +} + +model DdnetRecord { + mapId Int + timestamp DateTime + playerId Int + time Float + + @@id([mapId, timestamp, playerId]) + @@index([timestamp]) +} + +model DdnetOnlineDay { + day DateTime @db.Date + kind Int @db.SmallInt // 0 = country, 1 = mod + key String // location/mod code; 'ALL' = total + avgPlayers Int + maxPlayers Int + + @@id([day, kind, key]) +} + +model DdnetState { + key String @id + value Json +} diff --git a/libs/prisma/prisma/sql/countDdnetMapRecordsWithoutShared.sql b/libs/prisma/prisma/sql/countDdnetMapRecordsWithoutShared.sql new file mode 100644 index 0000000..f88ac11 --- /dev/null +++ b/libs/prisma/prisma/sql/countDdnetMapRecordsWithoutShared.sql @@ -0,0 +1,5 @@ +SELECT count(*) AS "count" +FROM "DdnetRaceBest" AS rb +JOIN "Player" AS p ON p."id" = rb."playerId" +WHERE rb."mapId" = $1 + AND (p."pollCount" < $2 OR p."occurrenceCount"::float8 < p."pollCount" * $3::float8); diff --git a/libs/prisma/prisma/sql/incrementDdnetPlayers.sql b/libs/prisma/prisma/sql/incrementDdnetPlayers.sql new file mode 100644 index 0000000..0226a5a --- /dev/null +++ b/libs/prisma/prisma/sql/incrementDdnetPlayers.sql @@ -0,0 +1,8 @@ +INSERT INTO "DdnetPlayer" ("playerId", "finishCount", "firstFinishAt", "lastFinishAt") +SELECT t."playerId", t."finishCount", t."firstFinishAt", t."lastFinishAt" +FROM unnest($1::integer[], $2::integer[], $3::timestamp[], $4::timestamp[]) + AS t("playerId", "finishCount", "firstFinishAt", "lastFinishAt") +ON CONFLICT ("playerId") DO UPDATE SET + "finishCount" = "DdnetPlayer"."finishCount" + EXCLUDED."finishCount", + "firstFinishAt" = LEAST("DdnetPlayer"."firstFinishAt", EXCLUDED."firstFinishAt"), + "lastFinishAt" = GREATEST("DdnetPlayer"."lastFinishAt", EXCLUDED."lastFinishAt"); diff --git a/libs/prisma/prisma/sql/incrementMapDayFinishes.sql b/libs/prisma/prisma/sql/incrementMapDayFinishes.sql new file mode 100644 index 0000000..d0040cd --- /dev/null +++ b/libs/prisma/prisma/sql/incrementMapDayFinishes.sql @@ -0,0 +1,5 @@ +INSERT INTO "MapDay" ("day", "mapId", "playTime", "playerCount", "finishCount") +SELECT t."day", t."mapId", 0, 0, t."finishCount" +FROM unnest($1::date[], $2::integer[], $3::integer[]) AS t("day", "mapId", "finishCount") +ON CONFLICT ("mapId", "day") DO UPDATE SET + "finishCount" = "MapDay"."finishCount" + EXCLUDED."finishCount"; diff --git a/libs/prisma/prisma/sql/incrementPlayerDayFinishes.sql b/libs/prisma/prisma/sql/incrementPlayerDayFinishes.sql new file mode 100644 index 0000000..2ddbe31 --- /dev/null +++ b/libs/prisma/prisma/sql/incrementPlayerDayFinishes.sql @@ -0,0 +1,5 @@ +INSERT INTO "PlayerDay" ("day", "playerId", "playTime", "finishCount") +SELECT t."day", t."playerId", 0, t."finishCount" +FROM unnest($1::date[], $2::integer[], $3::integer[]) AS t("day", "playerId", "finishCount") +ON CONFLICT ("day", "playerId") DO UPDATE SET + "finishCount" = "PlayerDay"."finishCount" + EXCLUDED."finishCount"; diff --git a/libs/prisma/prisma/sql/insertDdnetRecords.sql b/libs/prisma/prisma/sql/insertDdnetRecords.sql new file mode 100644 index 0000000..c6ab86a --- /dev/null +++ b/libs/prisma/prisma/sql/insertDdnetRecords.sql @@ -0,0 +1,5 @@ +INSERT INTO "DdnetRecord" ("mapId", "timestamp", "playerId", "time") +SELECT t."mapId", t."timestamp", t."playerId", t."time" +FROM unnest($1::integer[], $2::timestamp[], $3::integer[], $4::double precision[]) + AS t("mapId", "timestamp", "playerId", "time") +ON CONFLICT ("mapId", "timestamp", "playerId") DO NOTHING; diff --git a/libs/prisma/prisma/sql/listDdnetCurrentRecords.sql b/libs/prisma/prisma/sql/listDdnetCurrentRecords.sql new file mode 100644 index 0000000..098c159 --- /dev/null +++ b/libs/prisma/prisma/sql/listDdnetCurrentRecords.sql @@ -0,0 +1,4 @@ +-- The latest record event per map is the current record. +SELECT DISTINCT ON ("mapId") "mapId", "time" +FROM "DdnetRecord" +ORDER BY "mapId", "timestamp" DESC; diff --git a/libs/prisma/prisma/sql/listDdnetMapRecordsWithoutShared.sql b/libs/prisma/prisma/sql/listDdnetMapRecordsWithoutShared.sql new file mode 100644 index 0000000..3adfa38 --- /dev/null +++ b/libs/prisma/prisma/sql/listDdnetMapRecordsWithoutShared.sql @@ -0,0 +1,7 @@ +SELECT rb."time", rb."bestAt", p."name", p."pollCount", p."occurrenceCount" +FROM "DdnetRaceBest" AS rb +JOIN "Player" AS p ON p."id" = rb."playerId" +WHERE rb."mapId" = $1 + AND (p."pollCount" < $2 OR p."occurrenceCount"::float8 < p."pollCount" * $3::float8) +ORDER BY rb."time" ASC, rb."playerId" ASC +LIMIT $4 OFFSET $5; diff --git a/libs/prisma/prisma/sql/listDdnetMapsMissingThumb.sql b/libs/prisma/prisma/sql/listDdnetMapsMissingThumb.sql new file mode 100644 index 0000000..01e2557 --- /dev/null +++ b/libs/prisma/prisma/sql/listDdnetMapsMissingThumb.sql @@ -0,0 +1,7 @@ +SELECT m."mapId", map."name" +FROM "DdnetMap" AS m +JOIN "Map" AS map ON map."id" = m."mapId" +LEFT JOIN "DdnetMapThumb" AS t ON t."mapId" = m."mapId" +WHERE t."mapId" IS NULL +ORDER BY m."mapId" +LIMIT $1; diff --git a/libs/prisma/prisma/sql/refreshAllDdnetPoints.sql b/libs/prisma/prisma/sql/refreshAllDdnetPoints.sql new file mode 100644 index 0000000..84f46fc --- /dev/null +++ b/libs/prisma/prisma/sql/refreshAllDdnetPoints.sql @@ -0,0 +1,10 @@ +-- Full-table variant of refreshDdnetPoints: scans every best, so this runs +-- only at backfill tail and in the weekly full pass. +UPDATE "DdnetPlayer" p SET "points" = agg."points" +FROM ( + SELECT b."playerId", coalesce(sum(m."points"), 0)::int AS "points" + FROM "DdnetRaceBest" b + LEFT JOIN "DdnetMap" m ON m."mapId" = b."mapId" + GROUP BY b."playerId" +) agg +WHERE p."playerId" = agg."playerId" AND p."points" IS DISTINCT FROM agg."points"; diff --git a/libs/prisma/prisma/sql/refreshDdnetMapStats.sql b/libs/prisma/prisma/sql/refreshDdnetMapStats.sql new file mode 100644 index 0000000..03dfbc0 --- /dev/null +++ b/libs/prisma/prisma/sql/refreshDdnetMapStats.sql @@ -0,0 +1,44 @@ +-- Recomputes per-map aggregates from the bests for the given maps: total +-- finishes, median/quantiles of best times, the record run's splits, and the +-- per-checkpoint median split (0 where no best has data for that checkpoint). +WITH affected AS ( + SELECT unnest($1::integer[]) AS "mapId" +), +best_stats AS ( + SELECT b."mapId", + sum(b."finishCount")::int AS "finishCount", + percentile_cont(0.5) WITHIN GROUP (ORDER BY b."time") AS "medianTime", + (percentile_cont(ARRAY(SELECT g / 100.0 FROM generate_series(0, 100) g)) + WITHIN GROUP (ORDER BY b."time"))::real[] AS "timeQuantiles" + FROM "DdnetRaceBest" b + JOIN affected a ON a."mapId" = b."mapId" + GROUP BY b."mapId" +), +wr AS ( + SELECT DISTINCT ON (b."mapId") b."mapId", b."splits" + FROM "DdnetRaceBest" b + JOIN affected a ON a."mapId" = b."mapId" + ORDER BY b."mapId", b."time", b."playerId" +), +split_medians AS ( + SELECT s."mapId", array_agg(coalesce(s."median", 0)::real ORDER BY s."ord") AS "medianSplits" + FROM ( + SELECT b."mapId", u."ord", + percentile_cont(0.5) WITHIN GROUP (ORDER BY u."val") FILTER (WHERE u."val" > 0) AS "median" + FROM "DdnetRaceBest" b + JOIN affected a ON a."mapId" = b."mapId", + LATERAL unnest(b."splits") WITH ORDINALITY AS u("val", "ord") + GROUP BY b."mapId", u."ord" + ) s + GROUP BY s."mapId" +) +UPDATE "DdnetMap" m SET + "finishCount" = bs."finishCount", + "medianTime" = bs."medianTime", + "timeQuantiles" = bs."timeQuantiles", + "wrSplits" = coalesce(w."splits", '{}'), + "medianSplits" = coalesce(sm."medianSplits", '{}') +FROM best_stats bs +LEFT JOIN wr w ON w."mapId" = bs."mapId" +LEFT JOIN split_medians sm ON sm."mapId" = bs."mapId" +WHERE m."mapId" = bs."mapId"; diff --git a/libs/prisma/prisma/sql/refreshDdnetPoints.sql b/libs/prisma/prisma/sql/refreshDdnetPoints.sql new file mode 100644 index 0000000..411d103 --- /dev/null +++ b/libs/prisma/prisma/sql/refreshDdnetPoints.sql @@ -0,0 +1,11 @@ +-- Points = sum of map points over distinct finished maps. LEFT JOIN so a +-- player whose only pointed map was removed drops back to 0. +UPDATE "DdnetPlayer" p SET "points" = agg."points" +FROM ( + SELECT b."playerId", coalesce(sum(m."points"), 0)::int AS "points" + FROM "DdnetRaceBest" b + LEFT JOIN "DdnetMap" m ON m."mapId" = b."mapId" + WHERE b."playerId" = ANY($1::integer[]) + GROUP BY b."playerId" +) agg +WHERE p."playerId" = agg."playerId" AND p."points" IS DISTINCT FROM agg."points"; diff --git a/libs/prisma/prisma/sql/refreshDdnetPointsForMaps.sql b/libs/prisma/prisma/sql/refreshDdnetPointsForMaps.sql new file mode 100644 index 0000000..d35471f --- /dev/null +++ b/libs/prisma/prisma/sql/refreshDdnetPointsForMaps.sql @@ -0,0 +1,13 @@ +-- Recomputes points for every finisher of the given maps (used when a map's +-- points are re-rated or a map is added/removed). +UPDATE "DdnetPlayer" p SET "points" = agg."points" +FROM ( + SELECT b."playerId", coalesce(sum(m."points"), 0)::int AS "points" + FROM "DdnetRaceBest" b + LEFT JOIN "DdnetMap" m ON m."mapId" = b."mapId" + WHERE b."playerId" IN ( + SELECT DISTINCT f."playerId" FROM "DdnetRaceBest" f WHERE f."mapId" = ANY($1::integer[]) + ) + GROUP BY b."playerId" +) agg +WHERE p."playerId" = agg."playerId" AND p."points" IS DISTINCT FROM agg."points"; diff --git a/libs/prisma/prisma/sql/refreshDdnetPointsRanks.sql b/libs/prisma/prisma/sql/refreshDdnetPointsRanks.sql new file mode 100644 index 0000000..600e84e --- /dev/null +++ b/libs/prisma/prisma/sql/refreshDdnetPointsRanks.sql @@ -0,0 +1,8 @@ +-- Equal points share a rank; zero points means unranked (NULL). +UPDATE "DdnetPlayer" p SET "pointsRank" = r."rank" +FROM ( + SELECT "playerId", + CASE WHEN "points" > 0 THEN (rank() OVER (ORDER BY "points" DESC))::int END AS "rank" + FROM "DdnetPlayer" +) r +WHERE p."playerId" = r."playerId" AND p."pointsRank" IS DISTINCT FROM r."rank"; diff --git a/libs/prisma/prisma/sql/setMapDayFinishes.sql b/libs/prisma/prisma/sql/setMapDayFinishes.sql new file mode 100644 index 0000000..b9058d9 --- /dev/null +++ b/libs/prisma/prisma/sql/setMapDayFinishes.sql @@ -0,0 +1,5 @@ +INSERT INTO "MapDay" ("day", "mapId", "playTime", "playerCount", "finishCount") +SELECT t."day", t."mapId", 0, 0, t."finishCount" +FROM unnest($1::date[], $2::integer[], $3::integer[]) AS t("day", "mapId", "finishCount") +ON CONFLICT ("mapId", "day") DO UPDATE SET + "finishCount" = EXCLUDED."finishCount"; diff --git a/libs/prisma/prisma/sql/setPlayerDayFinishes.sql b/libs/prisma/prisma/sql/setPlayerDayFinishes.sql new file mode 100644 index 0000000..09315a2 --- /dev/null +++ b/libs/prisma/prisma/sql/setPlayerDayFinishes.sql @@ -0,0 +1,7 @@ +-- Authoritative recount for recent days: overwrites whatever is there, which +-- also restores counts wiped by a re-rollup of the same day. +INSERT INTO "PlayerDay" ("day", "playerId", "playTime", "finishCount") +SELECT t."day", t."playerId", 0, t."finishCount" +FROM unnest($1::date[], $2::integer[], $3::integer[]) AS t("day", "playerId", "finishCount") +ON CONFLICT ("day", "playerId") DO UPDATE SET + "finishCount" = EXCLUDED."finishCount"; diff --git a/libs/prisma/prisma/sql/upsertDdnetPartnerMaps.sql b/libs/prisma/prisma/sql/upsertDdnetPartnerMaps.sql new file mode 100644 index 0000000..0864563 --- /dev/null +++ b/libs/prisma/prisma/sql/upsertDdnetPartnerMaps.sql @@ -0,0 +1,10 @@ +INSERT INTO "DdnetPartnerMap" ("playerId", "partnerId", "mapId", "bestTime", "finishCount", "lastFinishAt") +SELECT t."playerId", t."partnerId", t."mapId", t."bestTime", t."finishCount", t."lastFinishAt" +FROM unnest( + $1::integer[], $2::integer[], $3::integer[], + $4::double precision[], $5::integer[], $6::date[] +) AS t("playerId", "partnerId", "mapId", "bestTime", "finishCount", "lastFinishAt") +ON CONFLICT ("playerId", "partnerId", "mapId") DO UPDATE SET + "finishCount" = "DdnetPartnerMap"."finishCount" + EXCLUDED."finishCount", + "bestTime" = LEAST("DdnetPartnerMap"."bestTime", EXCLUDED."bestTime"), + "lastFinishAt" = GREATEST("DdnetPartnerMap"."lastFinishAt", EXCLUDED."lastFinishAt"); diff --git a/libs/prisma/prisma/sql/upsertDdnetPartners.sql b/libs/prisma/prisma/sql/upsertDdnetPartners.sql new file mode 100644 index 0000000..e551f6d --- /dev/null +++ b/libs/prisma/prisma/sql/upsertDdnetPartners.sql @@ -0,0 +1,12 @@ +INSERT INTO "DdnetPartner" ("playerId", "partnerId", "finishCount", "bestTime", "bestMapId", "lastFinishAt") +SELECT t."playerId", t."partnerId", t."finishCount", t."bestTime", t."bestMapId", t."lastFinishAt" +FROM unnest( + $1::integer[], $2::integer[], $3::integer[], + $4::double precision[], $5::integer[], $6::date[] +) AS t("playerId", "partnerId", "finishCount", "bestTime", "bestMapId", "lastFinishAt") +ON CONFLICT ("playerId", "partnerId") DO UPDATE SET + "finishCount" = "DdnetPartner"."finishCount" + EXCLUDED."finishCount", + "bestMapId" = CASE WHEN EXCLUDED."bestTime" < "DdnetPartner"."bestTime" + THEN EXCLUDED."bestMapId" ELSE "DdnetPartner"."bestMapId" END, + "bestTime" = LEAST("DdnetPartner"."bestTime", EXCLUDED."bestTime"), + "lastFinishAt" = GREATEST("DdnetPartner"."lastFinishAt", EXCLUDED."lastFinishAt"); diff --git a/libs/prisma/prisma/sql/upsertDdnetPlayerNames.sql b/libs/prisma/prisma/sql/upsertDdnetPlayerNames.sql new file mode 100644 index 0000000..49983c1 --- /dev/null +++ b/libs/prisma/prisma/sql/upsertDdnetPlayerNames.sql @@ -0,0 +1,9 @@ +-- Finishes prove presence, so lastSeenAt may move forward but never back; +-- historical backfill rows therefore never touch players seen by polling. +INSERT INTO "Player" ("name", "lastSeenAt", "createdAt", "updatedAt") +SELECT t."name", t."lastSeenAt", now(), now() +FROM unnest($1::text[], $2::timestamp[]) AS t("name", "lastSeenAt") +ON CONFLICT ("name") DO UPDATE SET + "lastSeenAt" = EXCLUDED."lastSeenAt", + "updatedAt" = now() +WHERE EXCLUDED."lastSeenAt" > "Player"."lastSeenAt"; diff --git a/libs/prisma/prisma/sql/upsertDdnetRaceBests.sql b/libs/prisma/prisma/sql/upsertDdnetRaceBests.sql new file mode 100644 index 0000000..aae7744 --- /dev/null +++ b/libs/prisma/prisma/sql/upsertDdnetRaceBests.sql @@ -0,0 +1,18 @@ +-- $1: JSON-encoded array of {mapId, playerId, time, finishCount, +-- firstFinishAt, bestAt, splits} where splits is a (possibly empty) array of +-- seconds. LEAST/CASE merges keep the statement idempotent for the min fields. +INSERT INTO "DdnetRaceBest" ("mapId", "playerId", "time", "finishCount", "firstFinishAt", "bestAt", "splits") +SELECT t."mapId", t."playerId", t."time", t."finishCount", t."firstFinishAt", t."bestAt", + coalesce(ARRAY(SELECT value::real FROM jsonb_array_elements_text(t."splits")), '{}') +FROM jsonb_to_recordset(($1::text)::jsonb) AS t( + "mapId" integer, "playerId" integer, "time" double precision, + "finishCount" integer, "firstFinishAt" date, "bestAt" date, "splits" jsonb +) +ON CONFLICT ("mapId", "playerId") DO UPDATE SET + "finishCount" = "DdnetRaceBest"."finishCount" + EXCLUDED."finishCount", + "firstFinishAt" = LEAST("DdnetRaceBest"."firstFinishAt", EXCLUDED."firstFinishAt"), + "bestAt" = CASE WHEN EXCLUDED."time" < "DdnetRaceBest"."time" + THEN EXCLUDED."bestAt" ELSE "DdnetRaceBest"."bestAt" END, + "splits" = CASE WHEN EXCLUDED."time" < "DdnetRaceBest"."time" + THEN EXCLUDED."splits" ELSE "DdnetRaceBest"."splits" END, + "time" = LEAST("DdnetRaceBest"."time", EXCLUDED."time"); diff --git a/libs/prisma/prisma/sql/upsertDdnetTeamRacePlayers.sql b/libs/prisma/prisma/sql/upsertDdnetTeamRacePlayers.sql new file mode 100644 index 0000000..1951b3f --- /dev/null +++ b/libs/prisma/prisma/sql/upsertDdnetTeamRacePlayers.sql @@ -0,0 +1,5 @@ +-- $1: JSON-encoded array of {teamId (hex), playerId}. +INSERT INTO "DdnetTeamRacePlayer" ("teamId", "playerId") +SELECT decode(t."teamId", 'hex'), t."playerId" +FROM jsonb_to_recordset(($1::text)::jsonb) AS t("teamId" text, "playerId" integer) +ON CONFLICT ("teamId", "playerId") DO NOTHING; diff --git a/libs/prisma/prisma/sql/upsertDdnetTeamRaces.sql b/libs/prisma/prisma/sql/upsertDdnetTeamRaces.sql new file mode 100644 index 0000000..9066d86 --- /dev/null +++ b/libs/prisma/prisma/sql/upsertDdnetTeamRaces.sql @@ -0,0 +1,8 @@ +-- $1: JSON-encoded array of {teamId (hex), mapId, time, timestamp, size}. +INSERT INTO "DdnetTeamRace" ("teamId", "mapId", "time", "timestamp", "size") +SELECT decode(t."teamId", 'hex'), t."mapId", t."time", t."timestamp", t."size" +FROM jsonb_to_recordset(($1::text)::jsonb) AS t( + "teamId" text, "mapId" integer, "time" double precision, + "timestamp" timestamp, "size" smallint +) +ON CONFLICT ("teamId") DO NOTHING; diff --git a/libs/teerank/src/lib/bullmq/cleanAllQueues.ts b/libs/teerank/src/lib/bullmq/cleanAllQueues.ts index 7602c59..25b9d8f 100644 --- a/libs/teerank/src/lib/bullmq/cleanAllQueues.ts +++ b/libs/teerank/src/lib/bullmq/cleanAllQueues.ts @@ -9,6 +9,9 @@ import { cleanRankPlayerQueue } from "./queueRankPlayer"; import { cleanArchiveSnapshotsQueue } from "./queueArchiveSnapshots"; import { cleanRollupDayQueue } from "./queueRollupDay"; import { cleanRollupBackfillQueue } from "./queueRollupBackfill"; +import { cleanDdnetBackfillQueue } from "./queueDdnetBackfill"; +import { cleanDdnetImportQueue } from "./queueDdnetImport"; +import { cleanDdnetOnlineQueue } from "./queueDdnetOnline"; export async function cleanAllQueues() { await Promise.all([ @@ -23,5 +26,8 @@ export async function cleanAllQueues() { cleanArchiveSnapshotsQueue(), cleanRollupDayQueue(), cleanRollupBackfillQueue(), + cleanDdnetBackfillQueue(), + cleanDdnetImportQueue(), + cleanDdnetOnlineQueue(), ]); } diff --git a/libs/teerank/src/lib/bullmq/index.ts b/libs/teerank/src/lib/bullmq/index.ts index 523fbbd..defae36 100644 --- a/libs/teerank/src/lib/bullmq/index.ts +++ b/libs/teerank/src/lib/bullmq/index.ts @@ -11,3 +11,6 @@ export * from './queueUpdateGlobalCounts'; export * from './queueArchiveSnapshots'; export * from './queueRollupDay'; export * from './queueRollupBackfill'; +export * from './queueDdnetBackfill'; +export * from './queueDdnetImport'; +export * from './queueDdnetOnline'; diff --git a/libs/teerank/src/lib/bullmq/queueDdnetBackfill.ts b/libs/teerank/src/lib/bullmq/queueDdnetBackfill.ts new file mode 100644 index 0000000..3dbe87b --- /dev/null +++ b/libs/teerank/src/lib/bullmq/queueDdnetBackfill.ts @@ -0,0 +1,48 @@ +import { Job, Queue, Worker } from "bullmq"; +import { bullmqConnection } from "./config"; +import { hoursToSeconds } from "date-fns"; + +let ddnetBackfillQueue: Queue | null = null; + +const QUEUE_NAME_DDNET_BACKFILL = 'ddnet-backfill'; + +function getQueueDdnetBackfill() { + ddnetBackfillQueue ??= new Queue(QUEUE_NAME_DDNET_BACKFILL, { connection: bullmqConnection }); + return ddnetBackfillQueue; +} + +export async function scheduleDdnetBackfill() { + const queue = getQueueDdnetBackfill(); + await queue.add('ddnet-backfill-tick', {}, { + deduplication: { + id: 'ddnet-backfill-tick', + } + }); +} + +export async function processDdnetBackfillJobs(processor: () => Promise) { + const jobProcessor = async (_job: Job) => { + await processor(); + } + + return new Worker(QUEUE_NAME_DDNET_BACKFILL, jobProcessor, { + connection: bullmqConnection, + concurrency: 1, + removeOnComplete: { + age: hoursToSeconds(6), + }, + removeOnFail: { + count: 1000, + } + }); +} + +export async function cleanDdnetBackfillQueue() { + await getQueueDdnetBackfill().obliterate({ + force: true, + }); +} + +export async function getDdnetBackfillFailedCount() { + return getQueueDdnetBackfill().getFailedCount(); +} diff --git a/libs/teerank/src/lib/bullmq/queueDdnetImport.ts b/libs/teerank/src/lib/bullmq/queueDdnetImport.ts new file mode 100644 index 0000000..0152c82 --- /dev/null +++ b/libs/teerank/src/lib/bullmq/queueDdnetImport.ts @@ -0,0 +1,48 @@ +import { Job, Queue, Worker } from "bullmq"; +import { bullmqConnection } from "./config"; +import { hoursToSeconds } from "date-fns"; + +let ddnetImportQueue: Queue | null = null; + +const QUEUE_NAME_DDNET_IMPORT = 'ddnet-import'; + +function getQueueDdnetImport() { + ddnetImportQueue ??= new Queue(QUEUE_NAME_DDNET_IMPORT, { connection: bullmqConnection }); + return ddnetImportQueue; +} + +export async function scheduleDdnetImport() { + const queue = getQueueDdnetImport(); + await queue.add('ddnet-import', {}, { + deduplication: { + id: 'ddnet-import', + } + }); +} + +export async function processDdnetImportJobs(processor: () => Promise) { + const jobProcessor = async (_job: Job) => { + await processor(); + } + + return new Worker(QUEUE_NAME_DDNET_IMPORT, jobProcessor, { + connection: bullmqConnection, + concurrency: 1, + removeOnComplete: { + age: hoursToSeconds(48), + }, + removeOnFail: { + count: 1000, + } + }); +} + +export async function cleanDdnetImportQueue() { + await getQueueDdnetImport().obliterate({ + force: true, + }); +} + +export async function getDdnetImportFailedCount() { + return getQueueDdnetImport().getFailedCount(); +} diff --git a/libs/teerank/src/lib/bullmq/queueDdnetOnline.ts b/libs/teerank/src/lib/bullmq/queueDdnetOnline.ts new file mode 100644 index 0000000..f3a7887 --- /dev/null +++ b/libs/teerank/src/lib/bullmq/queueDdnetOnline.ts @@ -0,0 +1,48 @@ +import { Job, Queue, Worker } from "bullmq"; +import { bullmqConnection } from "./config"; +import { hoursToSeconds } from "date-fns"; + +let ddnetOnlineQueue: Queue | null = null; + +const QUEUE_NAME_DDNET_ONLINE = 'ddnet-online'; + +function getQueueDdnetOnline() { + ddnetOnlineQueue ??= new Queue(QUEUE_NAME_DDNET_ONLINE, { connection: bullmqConnection }); + return ddnetOnlineQueue; +} + +export async function scheduleDdnetOnline() { + const queue = getQueueDdnetOnline(); + await queue.add('ddnet-online', {}, { + deduplication: { + id: 'ddnet-online', + } + }); +} + +export async function processDdnetOnlineJobs(processor: () => Promise) { + const jobProcessor = async (_job: Job) => { + await processor(); + } + + return new Worker(QUEUE_NAME_DDNET_ONLINE, jobProcessor, { + connection: bullmqConnection, + concurrency: 1, + removeOnComplete: { + age: hoursToSeconds(6), + }, + removeOnFail: { + count: 1000, + } + }); +} + +export async function cleanDdnetOnlineQueue() { + await getQueueDdnetOnline().obliterate({ + force: true, + }); +} + +export async function getDdnetOnlineFailedCount() { + return getQueueDdnetOnline().getFailedCount(); +} diff --git a/package-lock.json b/package-lock.json index b2f7ec6..3ad8053 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,9 +19,11 @@ "@sentry/profiling-node": "^7.107.0", "@sentry/utils": "^7.107.0", "@types/lodash": "^4.17.13", + "@types/unzipper": "^0.10.11", "apache-arrow": "^21.2.0", "base64url": "^3.0.1", "bullmq": "^5.29.1", + "csv-parse": "^7.0.2", "date-fns": "^2.30.0", "fastify": "~4.13.0", "fastify-plugin": "~4.5.0", @@ -41,6 +43,7 @@ "sharp": "^0.32.6", "tailwind-merge": "^2.0.0", "tslib": "^2.3.0", + "unzipper": "^0.12.5", "zod": "^3.22.4" }, "devDependencies": { @@ -4172,6 +4175,13 @@ "node": ">=8" } }, + "node_modules/@nx-tools/core/node_modules/csv-parse": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@nx-tools/nx-prisma": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/@nx-tools/nx-prisma/-/nx-prisma-6.4.1.tgz", @@ -6635,6 +6645,15 @@ "integrity": "sha512-THo502dA5PzG/sfQH+42Lw3fvmYkceefOspdCwpHRul8ik2Jv1K8I5OZz1AT3/rs46kwgMCe9bSBmDLYkkOMGg==", "dev": true }, + "node_modules/@types/unzipper": { + "version": "0.10.11", + "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.11.tgz", + "integrity": "sha512-D25im2zjyMCcgL9ag6N46+wbtJBnXIr7SI4zHf9eJD2Dw2tEB5e+p5MYkrxKIVRscs5QV0EhtU9rgXSPx90oJg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/ws": { "version": "8.5.13", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz", @@ -8248,6 +8267,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -9100,8 +9125,7 @@ "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "node_modules/corser": { "version": "2.0.1", @@ -9516,10 +9540,10 @@ "dev": true }, "node_modules/csv-parse": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", - "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", - "dev": true + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-7.0.2.tgz", + "integrity": "sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==", + "license": "MIT" }, "node_modules/damerau-levenshtein": { "version": "1.0.8", @@ -10041,6 +10065,51 @@ "url": "https://dotenvx.com" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -14761,7 +14830,6 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dev": true, "dependencies": { "universalify": "^2.0.0" }, @@ -15832,8 +15900,7 @@ "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" }, "node_modules/node-machine-id": { "version": "1.1.12", @@ -17495,8 +17562,7 @@ "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "node_modules/process-warning": { "version": "2.3.2", @@ -20343,7 +20409,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "dev": true, "engines": { "node": ">= 10.0.0" } @@ -20357,6 +20422,33 @@ "node": ">= 0.8" } }, + "node_modules/unzipper": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.5.tgz", + "integrity": "sha512-tXYOi9R57Uj/2Z25SOs5RRSzq886MBQj2gY8dPL+xl/kv6s6SvByoKfAtvfVeEuhntWDgjd2o9p2lb4TVPAz0A==", + "license": "MIT", + "dependencies": { + "bluebird": "~3.7.2", + "duplexer2": "~0.1.4", + "fs-extra": "11.3.1", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" + } + }, + "node_modules/unzipper/node_modules/fs-extra": { + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.1.tgz", + "integrity": "sha512-eXvGGwZ5CL17ZSwHWd3bbgk7UUpF6IFHtP57NYYakPvHOs8GDgDe5KJI36jIJzDkJ6eJjuzRA8eBQb6SkKue0g==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/upath": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", diff --git a/package.json b/package.json index ec270cb..9a84132 100644 --- a/package.json +++ b/package.json @@ -70,9 +70,11 @@ "@sentry/profiling-node": "^7.107.0", "@sentry/utils": "^7.107.0", "@types/lodash": "^4.17.13", + "@types/unzipper": "^0.10.11", "apache-arrow": "^21.2.0", "base64url": "^3.0.1", "bullmq": "^5.29.1", + "csv-parse": "^7.0.2", "date-fns": "^2.30.0", "fastify": "~4.13.0", "fastify-plugin": "~4.5.0", @@ -92,6 +94,7 @@ "sharp": "^0.32.6", "tailwind-merge": "^2.0.0", "tslib": "^2.3.0", + "unzipper": "^0.12.5", "zod": "^3.22.4" } }