From b264f6f16fe895ecf9a29e97fb47aa8733106961 Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Sun, 10 Nov 2024 20:03:07 +0000 Subject: [PATCH 01/10] chore: lint --- components/icons.tsx | 352 ++++++++++++++++++------------------ components/primitives.ts | 86 ++++----- components/theme-switch.tsx | 134 +++++++------- config/fonts.ts | 8 +- types/data.d.ts | 2 +- types/index.ts | 2 +- 6 files changed, 293 insertions(+), 291 deletions(-) diff --git a/components/icons.tsx b/components/icons.tsx index e2a6989..59bfe35 100644 --- a/components/icons.tsx +++ b/components/icons.tsx @@ -3,212 +3,212 @@ import * as React from "react"; import { IconSvgProps } from "@/types"; export const Logo: React.FC = ({ - size = 36, - height, - ...props + size = 36, + height, + ...props }) => ( - - - + + + ); export const DiscordIcon: React.FC = ({ - size = 24, - width, - height, - ...props + size = 24, + width, + height, + ...props }) => { - return ( - - - - ); + return ( + + + + ); }; export const TwitterIcon: React.FC = ({ - size = 24, - width, - height, - ...props + size = 24, + width, + height, + ...props }) => { - return ( - - - - ); + return ( + + + + ); }; export const GithubIcon: React.FC = ({ - size = 24, - width, - height, - ...props + size = 24, + width, + height, + ...props }) => { - return ( - - - - ); + return ( + + + + ); }; export const MoonFilledIcon = ({ - size = 24, - width, - height, - ...props + size = 24, + width, + height, + ...props }: IconSvgProps) => ( - + ); export const SunFilledIcon = ({ - size = 24, - width, - height, - ...props + size = 24, + width, + height, + ...props }: IconSvgProps) => ( - + ); export const HeartFilledIcon = ({ - size = 24, - width, - height, - ...props + size = 24, + width, + height, + ...props }: IconSvgProps) => ( - + ); export const SearchIcon = (props: IconSvgProps) => ( - + ); export const NextUILogo: React.FC = (props) => { - const { width, height = 40 } = props; + const { width, height = 40 } = props; - return ( - - - - - - ); + return ( + + + + + + ); }; diff --git a/components/primitives.ts b/components/primitives.ts index 472973c..8fe6f52 100644 --- a/components/primitives.ts +++ b/components/primitives.ts @@ -1,53 +1,53 @@ import { tv } from "tailwind-variants"; export const title = tv({ - base: "tracking-tight inline font-semibold", - variants: { - color: { - violet: "from-[#FF1CF7] to-[#b249f8]", - yellow: "from-[#FF705B] to-[#FFB457]", - blue: "from-[#5EA2EF] to-[#0072F5]", - cyan: "from-[#00b7fa] to-[#01cfea]", - green: "from-[#6FEE8D] to-[#17c964]", - pink: "from-[#FF72E1] to-[#F54C7A]", - foreground: "dark:from-[#FFFFFF] dark:to-[#4B4B4B]", + base: "tracking-tight inline font-semibold", + variants: { + color: { + violet: "from-[#FF1CF7] to-[#b249f8]", + yellow: "from-[#FF705B] to-[#FFB457]", + blue: "from-[#5EA2EF] to-[#0072F5]", + cyan: "from-[#00b7fa] to-[#01cfea]", + green: "from-[#6FEE8D] to-[#17c964]", + pink: "from-[#FF72E1] to-[#F54C7A]", + foreground: "dark:from-[#FFFFFF] dark:to-[#4B4B4B]", + }, + size: { + sm: "text-3xl lg:text-4xl", + md: "text-[2.3rem] lg:text-5xl leading-9", + lg: "text-4xl lg:text-6xl", + }, + fullWidth: { + true: "w-full block", + }, }, - size: { - sm: "text-3xl lg:text-4xl", - md: "text-[2.3rem] lg:text-5xl leading-9", - lg: "text-4xl lg:text-6xl", + defaultVariants: { + size: "md", }, - fullWidth: { - true: "w-full block", - }, - }, - defaultVariants: { - size: "md", - }, - compoundVariants: [ - { - color: [ - "violet", - "yellow", - "blue", - "cyan", - "green", - "pink", - "foreground", - ], - class: "bg-clip-text text-transparent bg-gradient-to-b", - }, - ], + compoundVariants: [ + { + color: [ + "violet", + "yellow", + "blue", + "cyan", + "green", + "pink", + "foreground", + ], + class: "bg-clip-text text-transparent bg-gradient-to-b", + }, + ], }); export const subtitle = tv({ - base: "w-full md:w-1/2 my-2 text-lg lg:text-xl text-default-600 block max-w-full", - variants: { - fullWidth: { - true: "!w-full", + base: "w-full md:w-1/2 my-2 text-lg lg:text-xl text-default-600 block max-w-full", + variants: { + fullWidth: { + true: "!w-full", + }, + }, + defaultVariants: { + fullWidth: true, }, - }, - defaultVariants: { - fullWidth: true, - }, }); diff --git a/components/theme-switch.tsx b/components/theme-switch.tsx index 622c433..068f7aa 100644 --- a/components/theme-switch.tsx +++ b/components/theme-switch.tsx @@ -7,80 +7,82 @@ import clsx from "clsx"; import { SunFilledIcon, MoonFilledIcon } from "@/components/icons"; export interface ThemeSwitchProps { - className?: string; - classNames?: SwitchProps["classNames"]; + className?: string; + classNames?: SwitchProps["classNames"]; } export const ThemeSwitch: FC = ({ - className, - classNames, + className, + classNames, }) => { - const [isMounted, setIsMounted] = useState(false); + const [isMounted, setIsMounted] = useState(false); - const { theme, setTheme } = useTheme(); + const { theme, setTheme } = useTheme(); - const onChange = () => { - theme === "light" ? setTheme("dark") : setTheme("light"); - }; + const onChange = () => { + theme === "light" ? setTheme("dark") : setTheme("light"); + }; - const { - Component, - slots, - isSelected, - getBaseProps, - getInputProps, - getWrapperProps, - } = useSwitch({ - isSelected: theme === "light", - onChange, - }); + const { + Component, + slots, + isSelected, + getBaseProps, + getInputProps, + getWrapperProps, + } = useSwitch({ + isSelected: theme === "light", + onChange, + }); - useEffect(() => { - setIsMounted(true); - }, [isMounted]); + useEffect(() => { + setIsMounted(true); + }, [isMounted]); - // Prevent Hydration Mismatch - if (!isMounted) return
; + // Prevent Hydration Mismatch + if (!isMounted) return
; - return ( - - - - -
- {isSelected ? ( - - ) : ( - - )} -
- - ); + return ( + + + + +
+ {isSelected ? ( + + ) : ( + + )} +
+ + ); }; diff --git a/config/fonts.ts b/config/fonts.ts index 0e7d9c9..120c402 100644 --- a/config/fonts.ts +++ b/config/fonts.ts @@ -1,11 +1,11 @@ import { Fira_Code as FontMono, Inter as FontSans } from "next/font/google"; export const fontSans = FontSans({ - subsets: ["latin"], - variable: "--font-sans", + subsets: ["latin"], + variable: "--font-sans", }); export const fontMono = FontMono({ - subsets: ["latin"], - variable: "--font-mono", + subsets: ["latin"], + variable: "--font-mono", }); diff --git a/types/data.d.ts b/types/data.d.ts index cf9a8fb..4acd2a6 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -9,4 +9,4 @@ export interface ChannelData { subsApi: number; subsApiLastHit: number; }; -} \ No newline at end of file +} diff --git a/types/index.ts b/types/index.ts index cece4a4..f6db063 100644 --- a/types/index.ts +++ b/types/index.ts @@ -1,5 +1,5 @@ import { SVGProps } from "react"; export type IconSvgProps = SVGProps & { - size?: number; + size?: number; }; From 9c46f19821bc1909734acf82a2632464ff1e8bba Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Sun, 10 Nov 2024 22:09:08 +0000 Subject: [PATCH 02/10] fix: description --- config/site.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/site.ts b/config/site.ts index f559f7e..d90f045 100644 --- a/config/site.ts +++ b/config/site.ts @@ -2,8 +2,7 @@ export type SiteConfig = typeof siteConfig; export const siteConfig = { name: "JSALStats", - description: - "Make beautiful websites regardless of your design experience.", + description: "The home of YouTube analytics for JackSucksAtLife!", navItems: [ { label: "Home", From e99df2b1aef854fcdf344108202584b201c74f6d Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:32:24 +0000 Subject: [PATCH 03/10] feat: studio sub counts --- .env.example | 5 + .github/workflows/lockb.yml | 2 - pages/channel/[id].tsx | 342 ++++++++++++++++++++++++++++-------- 3 files changed, 273 insertions(+), 76 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8cf1ddd --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +POSTGRES_USER="" +POSTGRES_PASSWORD="" +POSTGRES_DB="" +POSTGRES_HOST="" +POSTGRES_PORT= \ No newline at end of file diff --git a/.github/workflows/lockb.yml b/.github/workflows/lockb.yml index 516d992..05ebba3 100644 --- a/.github/workflows/lockb.yml +++ b/.github/workflows/lockb.yml @@ -7,8 +7,6 @@ on: paths: - "**/package.json" - "**/bun.lockb" - pull_request: - types: [opened, reopened, synchronize] permissions: contents: write diff --git a/pages/channel/[id].tsx b/pages/channel/[id].tsx index a1f364f..2dbcf9c 100644 --- a/pages/channel/[id].tsx +++ b/pages/channel/[id].tsx @@ -12,9 +12,11 @@ interface PageState { hasError: boolean; isLoading: boolean; odometerSubs: number; + studioSubsChartOptions: any; analChartOptions: any; channelId: string; data: ChannelData | null; + channelIsStudio: boolean; } class IndexPage extends Component<{}, PageState> { @@ -29,7 +31,8 @@ class IndexPage extends Component<{}, PageState> { odometerSubs: 0, channelId: props.channelId, data: props.data, - analChartOptions: { + channelIsStudio: props.channelIsStudio, + studioSubsChartOptions: { chart: { backgroundColor: "transparent", type: "line", @@ -136,9 +139,173 @@ class IndexPage extends Component<{}, PageState> { }, ], }, + analChartOptions: { + chart: { + backgroundColor: "transparent", + type: "line", + zoomType: "x", + }, + title: { + text: "Subscribers", + style: { + color: "gray", + font: "Roboto Medium", + }, + }, + xAxis: { + type: "datetime", + tickPixelInterval: 150, + labels: { + style: { + color: "gray", + font: "Roboto Medium", + }, + }, + visible: true, + }, + yAxis: { + gridLineColor: "gray", + title: { + text: "", + }, + labels: { + style: { + color: "gray", + font: "Roboto Medium", + }, + }, + visible: true, + }, + plotOptions: { + series: { + threshold: null, + fillOpacity: 0.25, + animation: false, + lineWidth: 3, + }, + area: { + fillOpacity: 0.25, + }, + }, + credits: { + enabled: true, + text: "jsalstats.xyz", + href: "", + }, + time: { + useUTC: false, + }, + tooltip: { + shared: true, + formatter(this: any) { + if (!this.points || this.points.length === 0) return ""; + + const point = this.points[0]; + + const index = point.series.xData.indexOf(point.x); + const lastY = point.series.yData[index - 1]; + const dif = point.y - lastY; + + let r = + Highcharts.dateFormat( + "%A %b %e, %H:%M:%S", + new Date(point.x).getTime(), + ) + + '
\u25CF ' + + point.series.name + + ": " + + Number(point.y).toLocaleString(); + + if (dif < 0) { + r += + ' (' + + Number(dif).toLocaleString() + + ")"; + } + if (dif > 0) { + r += + ' (+' + + Number(dif).toLocaleString() + + ")"; + } + + return r; + }, + }, + series: [ + { + name: "Subscribers", + data: [], + showInLegend: false, + marker: { enabled: false }, + color: "#ff0000", + lineColor: "#ff0000", + lineWidth: 4, + type: "areaspline", + fillOpacity: 0.1, + }, + ], + }, }; } + fetchData = () => { + // TODO: https://github.com/JSALStats/site/issues/45 + if ( + this.state.channelIsStudio == false || + this.state.channelId == null || + (this.state.channelId != "UCrZKnWgOaYTTc7sc1KsVXZw" && + this.state.channelId != "UCUXNOmIdsoyd5fh5TZHHO5Q" && + this.state.channelId != "UCxLIJccyaRQDeyu6RzUsPuw") + ) { + return; + } else { + fetch(`https://studio.jsalstats.xyz/subcount`) + .then((response) => response.json()) + .then((data) => { + const subs = data[this.state.channelId]; + + // Update the chart data + this.setState((prevState) => { + const newDataPoint = [Date.now(), subs]; + let updatedData = [ + ...prevState.studioSubsChartOptions.series[0].data, + newDataPoint, + ]; + + if (updatedData.length > 1800) { + updatedData.shift(); + } + if (updatedData.length == 2) { + console.log(updatedData[1]); + if (updatedData[1][0] < updatedData[0][0] + 1000) { + updatedData.shift(); + } + } + + return { + odometerSubs: subs, + studioSubsChartOptions: { + ...prevState.studioSubsChartOptions, + series: [ + { + ...prevState.studioSubsChartOptions + .series[0], + data: updatedData, + }, + ], + }, + isLoading: false, + }; + }); + }) + .catch((error) => { + console.log(error); + this.setState({ isLoading: false }); + }); + } + }; + fetchAnal = async () => { if (this.state.channelId == null || this.state.data == null) { return; @@ -208,6 +375,8 @@ class IndexPage extends Component<{}, PageState> { componentDidMount() { this.fetchAnal(); + this.fetchData(); + setInterval(this.fetchData, 5000); } componentWillUnmount() { @@ -226,86 +395,111 @@ class IndexPage extends Component<{}, PageState> { return null; } - return ( - -
-
- {/* Banner */} -
- {this.state.data ? ( - User Avatar +
+
+ {/* Banner */} +
+ {this.state.data ? ( + User Avatar + ) : null} +
+

+ {this.state.data?.info.name.length > 30 + ? `${this.state.data?.info.name.slice(0, 30)}...` + : this.state.data?.info.name} +

+

+ {this.state.channelId} +

+
+
+
+
+
+ - ) : null} -
-

- {this.state.data?.info.name.length > 30 - ? `${this.state.data?.info.name.slice(0, 30)}...` - : this.state.data?.info.name} -

-

+

+ Subscribers +
+
+ + {/* Realtime Studio Subs */} + {this.state.channelIsStudio && ( +
+
- {this.state.channelId} + Studio Count +
+
+ +

+ Studio counts update every 5 seconds! +

+
+
+ )} + +
+
+ Analytics +
+
+ {this.state.analChartOptions && ( + + )} +

+ Data is still being added and collected :3

-
-
-
- -
-
- Subscribers -
-
- -
-
- Analytics -
-
- {this.state.analChartOptions && ( - - )} -

- Data is still being added and collected :3 -

-
-
-
- - ); +
+ + ); + } } } - export async function getServerSideProps(context: { query: { id: string } }) { const { id } = context.query; From 2afcd7239771c84b9f7dda0573b8a4284a234e42 Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Mon, 11 Nov 2024 21:48:22 +0000 Subject: [PATCH 04/10] feat: add subscriber counts to index.tsx closes #46 --- .env.example | 4 +- pages/index.tsx | 114 ++++++++++++++++++++--------------------- utils/abbreviateNum.ts | 11 ++++ 3 files changed, 69 insertions(+), 60 deletions(-) create mode 100644 utils/abbreviateNum.ts diff --git a/.env.example b/.env.example index 8cf1ddd..70c0e34 100644 --- a/.env.example +++ b/.env.example @@ -2,4 +2,6 @@ POSTGRES_USER="" POSTGRES_PASSWORD="" POSTGRES_DB="" POSTGRES_HOST="" -POSTGRES_PORT= \ No newline at end of file +POSTGRES_PORT= + +YOUTUBE_API_KEY="" \ No newline at end of file diff --git a/pages/index.tsx b/pages/index.tsx index 7983328..f847128 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -1,6 +1,4 @@ import { Link } from "@nextui-org/link"; -import { Snippet } from "@nextui-org/snippet"; -import { Code } from "@nextui-org/code"; import { button as buttonStyles } from "@nextui-org/theme"; import { SiYoutube, SiYoutubestudio } from "react-icons/si"; import Image from "next/image"; @@ -10,6 +8,7 @@ import { title, subtitle } from "@/components/primitives"; import { GithubIcon, TwitterIcon } from "@/components/icons"; import DefaultLayout from "@/layouts/default"; import { ChannelData } from "@/types/data"; +import { abbreviateNum } from "@/utils/abbreviateNum"; const fetchChannelData = async (id: string) => { const response = await fetch(`https://api.jsalstats.xyz/channel/${id}`); @@ -88,15 +87,6 @@ export default function IndexPage({
-
- - - Get started by editing{" "} - pages/index.tsx - - -
-

- {channels.studio.map((channel) => ( - - {channel.info.name} - {channel.info.name.length > 30 - ? `${channel.info.name.substring(0, 30)}...` - : channel.info.name} - - ))} + {channels.studio + .sort((a, b) => b.data.subsApi - a.data.subsApi) + .map((channel) => ( + + {channel.info.name} + {channel.info.name.length > 30 + ? `${channel.info.name.substring(0, 30)}...` + : channel.info.name}{" "} + • {abbreviateNum(channel.data.subsApi)} + + ))}

- {channels.nonstudio.map((channel) => ( - - {channel.info.name} - {channel.info.name.length > 30 - ? `${channel.info.name.substring(0, 30)}...` - : channel.info.name} - - ))} + {channels.nonstudio + .sort((a, b) => b.data.subsApi - a.data.subsApi) + .map((channel) => ( + + {channel.info.name} + {channel.info.name.length > 30 + ? `${channel.info.name.substring(0, 30)}...` + : channel.info.name}{" "} + • {abbreviateNum(channel.data.subsApi)} + + ))}

diff --git a/utils/abbreviateNum.ts b/utils/abbreviateNum.ts new file mode 100644 index 0000000..98a4a29 --- /dev/null +++ b/utils/abbreviateNum.ts @@ -0,0 +1,11 @@ +export function abbreviateNum(num: number) { + const units = ["", "K", "M", "B", "T"]; + let unitIndex = 0; + + while (num >= 1000 && unitIndex < units.length - 1) { + num /= 1000; + unitIndex++; + } + + return parseFloat(num.toFixed(3)) + units[unitIndex]; +} From 739b18a917b6918119107d2024c2794ac456c6c2 Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:59:20 +0000 Subject: [PATCH 05/10] update database --- server/db.ts | 94 +++++++++++++++++++++++++++++++++++++++++- server/getAllVideos.ts | 66 +++++++++++++++++++++++++++++ server/index.ts | 9 +++- 3 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 server/getAllVideos.ts diff --git a/server/db.ts b/server/db.ts index 5e379e6..9f5ba0f 100644 --- a/server/db.ts +++ b/server/db.ts @@ -8,6 +8,7 @@ const pool = new pg.Pool({ port: process.env.POSTGRES_PORT as unknown as number, }); +//#region Channels export async function createChannelTable() { const query = ` CREATE TABLE IF NOT EXISTS anal_channels ( @@ -17,11 +18,14 @@ export async function createChannelTable() { ); `; + const indexQuery = `CREATE INDEX IF NOT EXISTS idx_channel_id ON anal_channels(channel_id);`; + try { await pool.query(query); - console.log("Table created successfully"); + await pool.query(indexQuery); + console.log("Table and index created successfully"); } catch (err) { - console.error("Error creating table", err); + console.error("Error creating table or index", err); } } @@ -56,3 +60,89 @@ export async function getChannelData(channelId: string) { console.error("Error getting channel data", err); } } +//#endregion + +//#region Videos +export async function createVideoTable() { + const query = ` + CREATE TABLE IF NOT EXISTS anal_videos ( + video_id TEXT NOT NULL, + channel_id TEXT NOT NULL, + video_uploaded TIMESTAMP NOT NULL, + views INTEGER NOT NULL, + likes INTEGER NOT NULL, + comments INTEGER NOT NULL, + last_updated BIGINT NOT NULL, + video_is_studio BOOLEAN NOT NULL DEFAULT FALSE + ); + `; + + const indexQuery = ` + CREATE INDEX IF NOT EXISTS idx_video_id ON anal_videos(video_id); + CREATE INDEX IF NOT EXISTS idx_channel_id ON anal_videos(channel_id); + `; + + try { + await pool.query(query); + await pool.query(indexQuery); + console.log("Table and indices created successfully"); + } catch (err) { + console.error("Error creating table or indices", err); + } +} + +export async function createVideoHistoryTable() { + const query = ` + CREATE TABLE IF NOT EXISTS anal_video_history ( + video_id TEXT NOT NULL, + views INTEGER NOT NULL, + likes INTEGER NOT NULL, + comments INTEGER NOT NULL, + entry_added TIMESTAMP NOT NULL, + is_24hr BOOLEAN NOT NULL DEFAULT FALSE + ); + `; + + const indexQuery = ` + CREATE INDEX IF NOT EXISTS idx_video_id ON anal_video_history(video_id); + `; + + try { + await pool.query(query); + await pool.query(indexQuery); + console.log("Table and indices created successfully"); + } catch (err) { + console.error("Error creating table or indices", err); + } +} + +export async function insertNewVideo( + videoId: string, + channelId: string, + videoUploaded: number, + views: number, + likes: number, + comments: number, + lastUpdated: number, + videoIsStudio: boolean, +) { + const query = `INSERT INTO anal_videos (video_id, channel_id, video_uploaded, views, likes, comments, last_updated, video_is_studio) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`; + const values = [ + videoId, + channelId, + videoUploaded, + views, + likes, + comments, + lastUpdated, + videoIsStudio, + ]; + + try { + await pool.query(query, values); + console.log("Video inserted successfully"); + } catch (err) { + console.error("Error inserting video", err); + } +} +//#endregion diff --git a/server/getAllVideos.ts b/server/getAllVideos.ts new file mode 100644 index 0000000..db3fddd --- /dev/null +++ b/server/getAllVideos.ts @@ -0,0 +1,66 @@ +import { promises as fs } from "fs"; +import path from "path"; + +import { insertNewVideo } from "./db"; + +const filePath = path.join(process.cwd(), "public", "channels.json"); + +const data = await fs.readFile(filePath, "utf-8"); +const channels = JSON.parse(data).all; +let total = 0; + +for (const channelId of channels) { + console.log(`\nGetting all videos for ${channelId}`); + + let pageToken = ""; + + while (true) { + let url = `https://youtube.googleapis.com/youtube/v3/playlistItems?part=snippet,contentDetails&maxResults=50&playlistId=${channelId.replace("UC", "UU")}&key=${process.env.YOUTUBE_API_KEY}&pageToken=${pageToken}`; + + const res = await fetch(url); + const json = await res.json(); + + console.log(json.items.length, channelId, pageToken, total); + total += json.items.length; + + pageToken = json.nextPageToken; + + const videoIds = json.items + .map( + (item: { contentDetails: { videoId: any } }) => + item.contentDetails.videoId, + ) + .join(","); + + const statsUrl = `https://youtube.googleapis.com/youtube/v3/videos?part=statistics&id=${videoIds}&key=${process.env.YOUTUBE_API_KEY}`; + const statsRes = await fetch(statsUrl); + const statsJson = await statsRes.json(); + + for (const item of json.items) { + const stats = statsJson.items.find( + (stat: { id: any }) => stat.id === item.contentDetails.videoId, + ).statistics; + + await insertNewVideo( + item.contentDetails.videoId, + channelId, + item.snippet.publishedAt, + stats.viewCount || 0, + stats.likeCount || 0, + stats.commentCount || 0, + Date.now(), + [ + "UCrZKnWgOaYTTc7sc1KsVXZw", + "UCUXNOmIdsoyd5fh5TZHHO5Q", + "UCewMTclBJZPaNEfbf-qYMGA", + "UCxLIJccyaRQDeyu6RzUsPuw", + "UCd15dSPPT-EhTXekA7_UNAQ", + ].includes(channelId), + ); + } + + if (json.nextPageToken === undefined) { + break; + } + } +} diff --git a/server/index.ts b/server/index.ts index fb8c013..52017a1 100644 --- a/server/index.ts +++ b/server/index.ts @@ -3,7 +3,12 @@ import fs from "fs/promises"; import express from "express"; import cors from "cors"; -import { createChannelTable, getChannelData } from "./db"; +import { + createChannelTable, + createVideoHistoryTable, + createVideoTable, + getChannelData, +} from "./db"; const channelsData = JSON.parse( await fs.readFile("public/channels.json", "utf-8"), ); @@ -15,6 +20,8 @@ app.use(cors()); const port = process.env.PORT || 5816; await createChannelTable(); +await createVideoTable(); +await createVideoHistoryTable(); app.get("/channels", (req, res) => { res.status(200).json(channelsData); From 89c5e7218d53ef91393220635b7d21dd1a2b18f7 Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Thu, 14 Nov 2024 21:08:36 +0000 Subject: [PATCH 06/10] feat: new database stuff --- pages/channel/[id].tsx | 5 +- server/cron.ts | 123 +++++++++++++++++++++++++++++++++++++ server/db.ts | 136 +++++++++++++++++++++++++++++++++++++++-- server/index.ts | 4 ++ 4 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 server/cron.ts diff --git a/pages/channel/[id].tsx b/pages/channel/[id].tsx index 2dbcf9c..500f499 100644 --- a/pages/channel/[id].tsx +++ b/pages/channel/[id].tsx @@ -250,13 +250,10 @@ class IndexPage extends Component<{}, PageState> { } fetchData = () => { - // TODO: https://github.com/JSALStats/site/issues/45 if ( this.state.channelIsStudio == false || this.state.channelId == null || - (this.state.channelId != "UCrZKnWgOaYTTc7sc1KsVXZw" && - this.state.channelId != "UCUXNOmIdsoyd5fh5TZHHO5Q" && - this.state.channelId != "UCxLIJccyaRQDeyu6RzUsPuw") + !this.state.channelIsStudio ) { return; } else { diff --git a/server/cron.ts b/server/cron.ts new file mode 100644 index 0000000..467659d --- /dev/null +++ b/server/cron.ts @@ -0,0 +1,123 @@ +/** + * Note if you're using this code: + * The messageHandler.ts file is not included in this repo as it contains messages that are not meant to be public. + * Please remove the import statement below and all its calls if you're using this code. + * + * This repo is open-source so you can look at the code, however this file is not meant to be public + * as it's also shared with the studio project, which contains private information from Jack's Studio + */ +import cron from "cron"; + +import { + getAllChannelIds, + insertStudioChannel, + getAllVideoIds, + insertVideoData, + updateVideoData, + insertChannel, +} from "./db"; +// serverOnline(); + +const CronJob = cron.CronJob; +const jobChannels = new CronJob("*/5 * * * * *", updateChannels); +const jobChannelsStudio = new CronJob("0 * * * *", updateChannelsStudio); +const jobVideos = new CronJob("0 * * * *", updateAllVideos); + +jobChannels.start(); +jobChannelsStudio.start(); +jobVideos.start(); + +// updateChannels(); +// updateChannelsStudio(); +// updateAllVideos(); + +async function updateChannels() { + console.log("Updating channels"); + const channels = await getAllChannelIds(); + const channelIds = channels.map((channel) => channel.channel_id).join(","); + const data = await fetch( + `https://youtube.googleapis.com/youtube/v3/channels?part=statistics&id=${channelIds}&key=${process.env.YOUTUBE_API_KEY}`, + ).then((res) => res.json()); + + for (const channel of channels) { + const channelData = data.items.find( + (item: any) => item.id === channel.channel_id, + ); + const currentSubs = channelData?.statistics?.subscriberCount; + + console.log( + `Channel ID: ${channel.channel_id}, Old Subs: ${channel.subs_api}, New Subs: ${currentSubs}`, + ); + + if (currentSubs && currentSubs != channel.subs_api) { + insertChannel(channel.channel_id, currentSubs, Date.now()); + } + } +} + +async function updateChannelsStudio() { + const date = new Date(); + const channelIds = [ + "UCrZKnWgOaYTTc7sc1KsVXZw", + "UCUXNOmIdsoyd5fh5TZHHO5Q", + "UCxLIJccyaRQDeyu6RzUsPuw", + "UCd15dSPPT-EhTXekA7_UNAQ", + "UCewMTclBJZPaNEfbf-qYMGA", + ]; + + const data = await fetch("https://studio.jsalstats.xyz/subcount").then( + (res) => { + return res.json(); + }, + ); + + for (const channelId of channelIds) { + insertStudioChannel(channelId, data[channelId], date); + } +} + +async function updateAllVideos() { + const videos = await getAllVideoIds(); + const chunks: any = []; + + videos?.forEach((_video, index) => { + if (index % 50 === 0) { + chunks.push(videos.slice(index, index + 50)); + } + }); + + chunks.forEach(async (chunk: any) => { + const videoIds = chunk.join(","); + + await fetch( + `https://youtube.googleapis.com/youtube/v3/videos?part=statistics&id=${videoIds}&key=${process.env.YOUTUBE_API_KEY}`, + ) + .then((res) => res.json()) + .then((data) => { + data.items.forEach((item: any) => { + const videoId = item.id; + const views = item.statistics.viewCount || 0; + const likes = item.statistics.likeCount || 0; + const comments = item.statistics.commentCount || 0; + + console.log(videoId, views, likes, comments); + + insertVideoData( + videoId, + views, + likes, + comments, + new Date(), + false, + ); + updateVideoData( + videoId, + views, + likes, + comments, + new Date().getTime(), + ); + }); + }); + }); +} diff --git a/server/db.ts b/server/db.ts index 9f5ba0f..316e1d0 100644 --- a/server/db.ts +++ b/server/db.ts @@ -8,7 +8,7 @@ const pool = new pg.Pool({ port: process.env.POSTGRES_PORT as unknown as number, }); -//#region Channels +// TODO: Return type export async function createChannelTable() { const query = ` CREATE TABLE IF NOT EXISTS anal_channels ( @@ -29,6 +29,27 @@ export async function createChannelTable() { } } +// TODO: Return type +export async function createStudioChannelTable() { + const query = ` + CREATE TABLE IF NOT EXISTS anal_studio_channels ( + channel_id TEXT NOT NULL, + subs INTEGER NOT NULL, + time TIMESTAMP NOT NULL + ); + `; + const indexQuery = `CREATE INDEX IF NOT EXISTS idx_channel_id ON anal_studio_channels(channel_id);`; + + try { + await pool.query(query); + await pool.query(indexQuery); + console.log("Table and index created successfully"); + } catch (err) { + console.error("Error creating table or index", err); + } +} + +// TODO: Return type export async function insertChannel( channelId: string, subsApi: number, @@ -45,6 +66,24 @@ export async function insertChannel( } } +// TODO: Return type +export async function insertStudioChannel( + channelId: string, + subs: number, + time: Date, +) { + const query = `INSERT INTO anal_studio_channels (channel_id, subs, time) VALUES ($1, $2, $3)`; + const values = [channelId, subs, time]; + + try { + await pool.query(query, values); + console.log("Studio channel inserted successfully"); + } catch (err) { + console.error("Error inserting studio channel", err); + } +} + +// TODO: Return type export async function getChannelData(channelId: string) { const query = `SELECT subs_api, subs_api_hit FROM anal_channels WHERE channel_id = $1`; const values = [channelId]; @@ -60,9 +99,32 @@ export async function getChannelData(channelId: string) { console.error("Error getting channel data", err); } } -//#endregion -//#region Videos +export async function getAllChannelIds(): Promise< + { channel_id: string; subs_api: number }[] +> { + const query = ` + SELECT channel_id, subs_api + FROM anal_channels + WHERE (channel_id, subs_api_hit) IN ( + SELECT channel_id, MAX(subs_api_hit) + FROM anal_channels + GROUP BY channel_id + ) + `; + + try { + const res = await pool.query(query); + + return res.rows; + } catch (err) { + console.error("Error getting unique channel ids", err); + + return []; + } +} + +// TODO: Return type export async function createVideoTable() { const query = ` CREATE TABLE IF NOT EXISTS anal_videos ( @@ -91,6 +153,7 @@ export async function createVideoTable() { } } +// TODO: Return type export async function createVideoHistoryTable() { const query = ` CREATE TABLE IF NOT EXISTS anal_video_history ( @@ -116,6 +179,7 @@ export async function createVideoHistoryTable() { } } +// TODO: Return type export async function insertNewVideo( videoId: string, channelId: string, @@ -145,4 +209,68 @@ export async function insertNewVideo( console.error("Error inserting video", err); } } -//#endregion + +// TODO: Return type +export async function insertVideoData( + videoId: string, + views: number, + likes: number, + comments: number, + entryAdded: Date, + is24hr: boolean, +) { + const query = `INSERT INTO anal_video_history (video_id, views, likes, comments, entry_added, is_24hr) VALUES ($1, $2, $3, $4, $5, $6)`; + const values = [videoId, views, likes, comments, entryAdded, is24hr]; + + try { + await pool.query(query, values); + // console.log("Video history inserted successfully"); + } catch (err) { + console.error("Error inserting video history", err); + } +} + +// TODO: Return type +export async function getAllStudioVideos() { + const query = `SELECT * FROM anal_videos WHERE video_is_studio = TRUE`; + + try { + const res = await pool.query(query); + + return res.rows; + } catch (err) { + console.error("Error getting studio videos", err); + } +} + +// TODO: Return type +export async function getAllVideoIds() { + const query = `SELECT video_id FROM anal_videos`; + + try { + const res = await pool.query(query); + + return res.rows.map((row) => row.video_id); + } catch (err) { + console.error("Error getting video ids", err); + } +} + +// TODO: Return type +export async function updateVideoData( + videoId: string, + views: number, + likes: number, + comments: number, + lastUpdated: number, +) { + const query = `UPDATE anal_videos SET views = $1, likes = $2, comments = $3, last_updated = $4 WHERE video_id = $5`; + const values = [views, likes, comments, lastUpdated, videoId]; + + try { + await pool.query(query, values); + console.log("Video updated successfully"); + } catch (err) { + console.error("Error updating video", err); + } +} diff --git a/server/index.ts b/server/index.ts index 52017a1..7b9552c 100644 --- a/server/index.ts +++ b/server/index.ts @@ -5,6 +5,7 @@ import cors from "cors"; import { createChannelTable, + createStudioChannelTable, createVideoHistoryTable, createVideoTable, getChannelData, @@ -20,9 +21,12 @@ app.use(cors()); const port = process.env.PORT || 5816; await createChannelTable(); +await createStudioChannelTable(); await createVideoTable(); await createVideoHistoryTable(); +import "./cron"; + app.get("/channels", (req, res) => { res.status(200).json(channelsData); }); From d0f4da8cc485ef02805cf23c14b79f31b05f5fa6 Mon Sep 17 00:00:00 2001 From: GalvinPython <77013913+GalvinPython@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:52:16 +0000 Subject: [PATCH 07/10] idk what i did, im just updating this so i can code from my server without conflicts --- next.config.js | 17 ++ package.json | 2 +- pages/404.tsx | 1 - pages/_document.tsx | 1 + pages/api/video/[videoId].ts | 9 +- pages/channel/[id].tsx | 7 +- pages/index.tsx | 2 - pages/video/[id].tsx | 504 +++++++++++++++++++++++++++++++++++ server/cron.ts | 13 +- server/db.ts | 35 ++- server/index.ts | 31 +++ 11 files changed, 599 insertions(+), 23 deletions(-) create mode 100644 pages/video/[id].tsx diff --git a/next.config.js b/next.config.js index dcc13ca..fdbd092 100644 --- a/next.config.js +++ b/next.config.js @@ -19,6 +19,23 @@ const nextConfig = { }, ]; }, + async headers() { + return [ + { + source: '/(.*)', + headers: [ + { + key: 'Access-Control-Allow-Origin', + value: '*', + }, + { + key: 'Access-Control-Allow-Methods', + value: 'GET', + }, + ], + }, + ]; + }, }; module.exports = nextConfig; diff --git a/package.json b/package.json index 4805f6a..1f79aff 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "concurrently 'next dev --turbo --port 5815' 'bun --watch ./server/index.ts'", "build": "next build", - "start:site": "next start --port 5815", + "start:site": "next start --port 5817", "start:server": "bun ./server/index.ts", "lint": "eslint . --ext .ts,.tsx -c .eslintrc.json --fix" }, diff --git a/pages/404.tsx b/pages/404.tsx index 85f357d..f27d329 100644 --- a/pages/404.tsx +++ b/pages/404.tsx @@ -19,7 +19,6 @@ export default function IndexPage() {
+ {/*