Added cached stats page

2022-01-15 01:59:06 +01:00 · 2022-01-15 01:59:06 +01:00 · 99783b4728
commit 99783b4728
--- a/1
+++ b/1
@ -2,6 +2,7 @@ FROM node:16-bullseye AS build
 ENV POSTGRES_URL='postgresql://fedisearch:passwd@postgres:5432/fedisearch?schema=public' \
    MATOMO_URL='' \
    MATOMO_SITE_ID='' \
+    STATS_CACHE_MINUTES=60 \
    TZ='UTC'
 WORKDIR /srv
 COPY application/package*.json ./
--- a/README.md
+++ b/README.md
@ -10,11 +10,12 @@ Only fulltext search is currently supported. More precise filtering is planned f

 Configuration is done using environmental variables:

-| Variable         | Description                                                                                                        | Value example                                                           |
-|------------------|--------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|
-| `POSTGRES_URL`   | Postgres database uri                                                                                              | `postgresql://fedisearch:passwd@postgres:5432/fedisearch?schema=public` |
-| `MATOMO_URL`     | Optional url of Matomo server for collecting usage statistics. Leaving it empty disables collecting analytics.     | `https://matomo.myserver.tld`                                           |
-| `MATOMO_SITE_ID` | Optional Matomo site id parameter for collecting usage statistics. Leaving it empty disables collecting analytics. | `8`                                                                     |
+| Variable              | Description                                                                                                        | Value example                                                           |
+|-----------------------|--------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|
+| `POSTGRES_URL`        | Postgres database uri                                                                                              | `postgresql://fedisearch:passwd@postgres:5432/fedisearch?schema=public` |
+| `MATOMO_URL`          | Optional url of Matomo server for collecting usage statistics. Leaving it empty disables collecting analytics.     | `https://matomo.myserver.tld`                                           |
+| `MATOMO_SITE_ID`      | Optional Matomo site id parameter for collecting usage statistics. Leaving it empty disables collecting analytics. | `8`                                                                     |
+| `STATS_CACHE_MINUTES` | Optional number of minutes to cache heavily calculated stats data                                                  | `60`                                                                    |

 ## Deploy

--- a/application/package-lock.json
+++ b/application/package-lock.json
--- a/application/package.json
+++ b/application/package.json
@ -25,7 +25,8 @@
    "sass": "^1.45.1",
    "striptags": "^3.2.0",
    "typescript-collections": "^1.3.3",
-    "zod": "^3.11.6"
+    "zod": "^3.11.6",
+    "node-cache": "^5.1.2"
  },
  "devDependencies": {
    "@next/eslint-plugin-next": "^12.0.7",
--- a/application/src/components/Layout.tsx
+++ b/application/src/components/Layout.tsx
@ -3,11 +3,12 @@ import Head from 'next/head'
 import Footer from './Footer'
 import getMatomo from '../lib/getMatomo'
 import { UserOptions } from '@datapunt/matomo-tracker-js/es/types'
+import NavItem from './NavItem'

 export const siteTitle = 'FediSearch'
 export const siteDescription = 'Search people on Fediverse'

-const Layout:React.FC<{ matomoConfig:UserOptions, children: React.ReactNode }> = ({ matomoConfig, children }) => {
+const Layout: React.FC<{ matomoConfig: UserOptions, children: React.ReactNode }> = ({ matomoConfig, children }) => {
  useEffect(() => {
    getMatomo(matomoConfig).trackPageView()
  }, [])
@ -32,6 +33,27 @@ const Layout:React.FC<{ matomoConfig:UserOptions, children: React.ReactNode }> =
                    />
                </a>
            </header>
+            <nav>
+                <ul>
+                    <NavItem path={'/'} label={'Search people'} icon={(
+                        <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="user"
+                             className="svg-inline--fa fa-user fa-w-14" role="img" xmlns="http://www.w3.org/2000/svg"
+                             viewBox="0 0 448 512">
+                            <path fill="currentColor"
+                                  d="M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"/>
+                        </svg>
+                    )} />
+                    <NavItem path={'/stats'} label={'Index stats'} icon={(
+
+                        <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="chart-pie"
+                             className="svg-inline--fa fa-chart-pie fa-w-17" role="img"
+                             xmlns="http://www.w3.org/2000/svg" viewBox="0 0 544 512">
+                            <path fill="currentColor"
+                                  d="M527.79 288H290.5l158.03 158.03c6.04 6.04 15.98 6.53 22.19.68 38.7-36.46 65.32-85.61 73.13-140.86 1.34-9.46-6.51-17.85-16.06-17.85zm-15.83-64.8C503.72 103.74 408.26 8.28 288.8.04 279.68-.59 272 7.1 272 16.24V240h223.77c9.14 0 16.82-7.68 16.19-16.8zM224 288V50.71c0-9.55-8.39-17.4-17.84-16.06C86.99 51.49-4.1 155.6.14 280.37 4.5 408.51 114.83 513.59 243.03 511.98c50.4-.63 96.97-16.87 135.26-44.03 7.9-5.6 8.42-17.23 1.57-24.08L224 288z"/>
+                        </svg>
+                    )} />
+                </ul>
+            </nav>
            <main>
                {children}
            </main>
--- a/application/src/components/Loader.tsx
+++ b/application/src/components/Loader.tsx
@ -1,10 +1,10 @@
 import React, { ReactNode } from 'react'

-const Loader:React.FC<{ children: ReactNode, loading: boolean }> = ({ children, loading }) => {
+const Loader:React.FC<{ children: ReactNode, loading: boolean, hideContent?:boolean }> = ({ hideContent, children, loading }) => {
  return (
        <div className={'loader' + (loading ? ' -loading' : '')}>
            <div className={'loader-content'}>
-                {children}
+                { hideContent && loading ? '' : children}
            </div>
            <div className={'loader-visualisation'}>
            {loading
--- a/application/src/components/NavItem.tsx
+++ b/application/src/components/NavItem.tsx
@ -0,0 +1,18 @@
+import React, { FC, ReactElement } from 'react'
+import Link from 'next/link'
+import { useRouter } from 'next/router'
+
+const NavItem:FC<{path:string, label:string, icon:ReactElement}> = ({ path, label, icon }) => {
+  const router = useRouter()
+  return (
+        <li>
+            <Link href={path}>
+                <a className={router.pathname === path ? 'active' : ''}>
+                    {icon}
+                    <span>{label}</span>
+                </a>
+            </Link>
+        </li>
+  )
+}
+export default NavItem
--- a/application/src/lib/cache.ts
+++ b/application/src/lib/cache.ts
@ -0,0 +1,3 @@
+import NodeCache from 'node-cache'
+
+export const cache = new NodeCache()
--- a/application/src/pages/api/stats.ts
+++ b/application/src/pages/api/stats.ts
@ -0,0 +1,64 @@
+import prisma from '../../lib/prisma'
+import { NextApiRequest, NextApiResponse } from 'next'
+import { StatsResponse, StatsResponseSoftware } from '../../types/StatsResponse'
+import { cache } from '../../lib/cache'
+
+interface StatsItem {
+    softwarename: string | null,
+    nodecount: number,
+    accountcount: number,
+    channelcount: number,
+    newnodescount: number
+}
+
+const CACHE_KEY = 'stats'
+
+const handleGetStats = async (req: NextApiRequest, res: NextApiResponse<StatsResponse>): Promise<void> => {
+  if (!cache.has(CACHE_KEY)) {
+    console.info('Retrieving new stats')
+    const data = await prisma.$queryRaw<StatsItem[]>`
+          select n."softwareName" as softwarename,
+                 count(n.id)      as nodecount,
+                 (
+                     select count("Feed".id)
+                     from "Feed"
+                              join "Node" on "Feed"."nodeId" = "Node".id and "Node"."softwareName" = n."softwareName"
+                     where "Feed".type = 'account'
+                 )                as accountcount,
+                 (
+                     select count("Feed".id)
+                     from "Feed"
+                              join "Node" on "Feed"."nodeId" = "Node".id and "Node"."softwareName" = n."softwareName"
+                     where "Feed".type = 'channel'
+                 )                as channelcount,
+                 (
+                     select count("Node".id)
+                     from "Node"
+                     where "Node"."refreshedAt" IS Null
+                       and "Node"."softwareName" = n."softwareName"
+                 )                as newnodescount
+          from "Node" n
+          group by n."softwareName"
+          having count(n.id) > 1
+          order by nodecount desc;
+      `
+    cache.set<StatsResponse>(CACHE_KEY, {
+      softwares: data.map(
+        (item: StatsItem): StatsResponseSoftware => {
+          return {
+            name: item.softwarename,
+            nodeCount: item.nodecount,
+            accountCount: item.accountcount,
+            channelCount: item.channelcount,
+            newNodeCount: item.newnodescount
+          }
+        }
+      )
+    }, parseInt(process.env.STATS_CACHE_MINUTES ?? '60') * 60 * 1000)
+  }
+  console.info('Returning stats from cache')
+  res.status(200)
+    .json(cache.get<StatsResponse>(CACHE_KEY))
+}
+
+export default handleGetStats
--- a/application/src/pages/index.tsx
+++ b/application/src/pages/index.tsx
@ -108,30 +108,29 @@ const Home:React.FC<InferGetServerSidePropsType<typeof getServerSideProps>> = ({
            <Head>
                <title>{siteTitle}</title>
            </Head>
-            <nav>
-                <form onSubmit={handleSearchSubmit}>
-                    <label htmlFor={'query'}>Search on fediverse</label>
-                    <input
-                        name={'query'}
-                        id={'query'}
-                        type={'search'}
-                        onChange={handleQueryChange}
-                        onBlur={handleQueryChange}
-                        value={query}
-                        placeholder={'Search on fediverse'}
-                    />
-                    <button type={'submit'}>
-                        <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="search"
-                             className="svg-inline--fa fa-search fa-w-16" role="img" xmlns="http://www.w3.org/2000/svg"
-                             viewBox="0 0 512 512">
-                            <path fill="currentColor"
-                                  d="M505 442.7L405.3 343c-4.5-4.5-10.6-7-17-7H372c27.6-35.3 44-79.7 44-128C416 93.1 322.9 0 208 0S0 93.1 0 208s93.1 208 208 208c48.3 0 92.7-16.4 128-44v16.3c0 6.4 2.5 12.5 7 17l99.7 99.7c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.6.1-34zM208 336c-70.7 0-128-57.2-128-128 0-70.7 57.2-128 128-128 70.7 0 128 57.2 128 128 0 70.7-57.2 128-128 128z"></path>
-                            <title>Search</title>
-                        </svg>
-                        <span>Search</span>
-                    </button>
-                </form>
-            </nav>
+            <form onSubmit={handleSearchSubmit}>
+              <label htmlFor={'query'}>Search on fediverse</label>
+              <input
+                  name={'query'}
+                  id={'query'}
+                  type={'search'}
+                  onChange={handleQueryChange}
+                  onBlur={handleQueryChange}
+                  value={query}
+                  placeholder={'Search on fediverse'}
+                  autoFocus={true}
+              />
+              <button type={'submit'}>
+                <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="search"
+                     className="svg-inline--fa fa-search fa-w-16" role="img" xmlns="http://www.w3.org/2000/svg"
+                     viewBox="0 0 512 512">
+                  <path fill="currentColor"
+                        d="M505 442.7L405.3 343c-4.5-4.5-10.6-7-17-7H372c27.6-35.3 44-79.7 44-128C416 93.1 322.9 0 208 0S0 93.1 0 208s93.1 208 208 208c48.3 0 92.7-16.4 128-44v16.3c0 6.4 2.5 12.5 7 17l99.7 99.7c9.4 9.4 24.6 9.4 33.9 0l28.3-28.3c9.4-9.4 9.4-24.6.1-34zM208 336c-70.7 0-128-57.2-128-128 0-70.7 57.2-128 128-128 70.7 0 128 57.2 128 128 0 70.7-57.2 128-128 128z" />
+                  <title>Search</title>
+                </svg>
+                <span>Search</span>
+              </button>
+            </form>
            <Loader loading={loading}>
                {
                    loaded
--- a/application/src/pages/stats.tsx
+++ b/application/src/pages/stats.tsx
@ -0,0 +1,99 @@
+import Head from 'next/head'
+import Layout, { siteTitle } from '../components/Layout'
+import { matomoConfig } from '../lib/matomoConfig'
+import { GetServerSideProps, InferGetServerSidePropsType } from 'next'
+import React, { useEffect, useState } from 'react'
+import Loader from '../components/Loader'
+import axios from 'axios'
+import { StatsResponse, statsResponseSchema } from '../types/StatsResponse'
+import SoftwareBadge from '../components/badges/SoftwareBadge'
+
+const Stats: React.FC<InferGetServerSidePropsType<typeof getServerSideProps>> = ({ matomoConfig }) => {
+  const [loading, setLoading] = useState<boolean>(true)
+  const [stats, setStats] = useState<StatsResponse|null>(null)
+
+  const loadStats = async () => {
+    try {
+      const response = await axios.get('/api/stats')
+      const stats = await statsResponseSchema.parseAsync(response.data)
+      setStats(stats)
+    } catch (err) {
+      setStats(null)
+      console.log(err)
+    }
+    setLoading(false)
+  }
+  useEffect(() => {
+    loadStats()
+  }, [])
+  const summary = {
+    nodeCount: 0,
+    accountCount: 0,
+    channelCount: 0
+  }
+  return (
+        <Layout matomoConfig={matomoConfig}>
+            <Head>
+                <title>{'Stats | ' + siteTitle}</title>
+            </Head>
+            <h1>Index stats</h1>
+            <Loader loading={loading} hideContent={true}>
+                {stats === null
+                  ? (<p>Failed to load stats data!</p>)
+                  : (
+                      <>
+                      <table>
+                            <thead>
+                            <tr>
+                                <th>Software</th>
+                                <th className={'number-cell'}>Instance count</th>
+                                <th className={'number-cell'}>Account count</th>
+                                <th className={'number-cell'}>Channel count</th>
+                            </tr>
+                            </thead>
+                            <tbody>
+                            {
+                                stats.softwares.map((software, index) => {
+                                  summary.nodeCount += software.nodeCount
+                                  summary.accountCount += software.accountCount
+                                  summary.channelCount += software.channelCount
+                                  return (
+                                        <tr key={index}>
+                                            <td>{software.name !== null
+                                              ? <SoftwareBadge softwareName={software.name}/>
+                                              : <em>Not recognized</em>}</td>
+                                            <td className={'number-cell'}>{software.nodeCount}</td>
+                                            <td className={'number-cell'}>{software.accountCount}</td>
+                                            <td className={'number-cell'}>{software.channelCount}</td>
+                                        </tr>
+                                  )
+                                })
+                            }
+                            </tbody>
+                            <tfoot>
+                            <tr>
+                                <th>Summary</th>
+                                <th className={'number-cell'}>{summary.nodeCount}</th>
+                                <th className={'number-cell'}>{summary.accountCount}</th>
+                                <th className={'number-cell'}>{summary.channelCount}</th>
+                            </tr>
+                            </tfoot>
+                        </table>
+                      </>
+                    )
+                }
+            </Loader>
+        </Layout>
+  )
+}
+
+export const getServerSideProps: GetServerSideProps = async (context) => {
+  console.info('Loading matomo config', matomoConfig)
+  return {
+    props: {
+      matomoConfig
+    }
+  }
+}
+
+export default Stats
--- a/application/src/styles/global.scss
+++ b/application/src/styles/global.scss
@ -6,14 +6,38 @@
  --accent-color: #00a2fe;
  --front-bg-color: #3a3c40;
  --front-fg-color: rgb(156, 156, 156);
-  @media (prefers-color-scheme: light) {
-    --main-bg-color: #ededed;
-    --main-fg-color: #212224;
-    --front-bg-color: rgb(204, 204, 204);
-    --front-fg-color:  #3a3c40;
-  }
+  //@media (prefers-color-scheme: light) {
+  //  --main-bg-color: #ededed;
+  //  --main-fg-color: #212224;
+  //  --front-bg-color: rgb(204, 204, 204);
+  //  --front-fg-color: #3a3c40;
+  //}
 }

+table{
+  width: 100%;
+  text-align: left;
+  img{
+    max-width: 1em;
+    max-height: 1em;
+    display: inline;
+    margin-right: 0.5em;
+  }
+
+  td, th {
+    border: 1px solid var(--front-bg-color);
+    border-radius: 0.3em;
+    padding: 0.3em;
+    word-break: break-word;
+    &.number-cell{
+      text-align: right;
+    }
+  }
+
+  th {
+    background-color: var(--front-bg-color);
+  }
+}

 input,
 textarea,
@ -72,7 +96,8 @@ body {
  font-size: 18px;
  background-color: var(--main-bg-color);
  color: var(--main-fg-color);
-  &>div{
+
+  & > div {
    width: 100%;
  }
 }
@ -134,7 +159,8 @@ a:hover {
    .logo {
      grid-column: start/ text;
      grid-row: start/end;
-      img{
+
+      img {
        height: 100%;
      }
    }
@ -181,6 +207,60 @@ form {
  }
 }

+nav {
+  ul {
+    padding: 0;
+    margin: 0;
+    li {
+      display: inline;
+      margin: 2px;
+      padding: 0;
+      list-style: none;
+      a {
+        padding: .5em;
+        background-color: var(--front-fg-color);
+        color: var(--main-bg-color);
+        white-space: nowrap;
+        box-shadow: var(--shadow);
+        &:hover{
+          outline: 1px solid var(--front-fg-color);
+          text-decoration: none;
+          outline-offset: 2px;
+        }
+        &:active,
+        &.active{
+          font-weight: bold;
+          background-color: var(--front-fg-color);
+          color: var(--front-bg-color);
+          svg{
+            max-width: 1.5em;
+            max-height: 1.5em;
+            position: relative;
+            top: 0.25em;
+          }
+        }
+        svg{
+          max-width: 1em;
+          max-height: 1em;
+          margin-right: 0.5em;
+        }
+      }
+      &:first-child{
+        a{
+          border-bottom-left-radius: 2em;
+          border-top-left-radius: 2em;
+        }
+      }
+      &:last-child{
+        a{
+          border-bottom-right-radius: 2em;
+          border-top-right-radius: 2em;
+        }
+      }
+    }
+  }
+}
+
 .loader {
  .loader-visualisation {
    margin: 1em 0;
@ -246,13 +326,13 @@ img {
    border-radius: 0.5em;
    width: 100%;
    grid-column: start /main;
-    grid-row: span 3 ;
+    grid-row: span 3;
  }


  .display-name {
    grid-column: main / end;
-    grid-row: start ;
+    grid-row: start;
    margin: 0;
    word-break: break-word;
  }
@ -263,13 +343,15 @@ img {
    word-break: break-word;
  }

-  .badges{
+  .badges {
    grid-column: main /end;
    grid-row: badges;
-    .label{
+
+    .label {
      display: none;
    }
-    img,svg{
+
+    img, svg {
      width: 100%;
      max-width: 1em;
      max-height: 1em;
@ -279,17 +361,18 @@ img {
      fill: var(--main-fg-color);
    }

-    .badge{
+    .badge {
      display: inline-block;
      margin-right: 1em;
      white-space: nowrap;
    }

    .subscriptions {
-      &>*{
+      & > * {
        display: inline;
        margin-right: 0.3em;
-        &:last-child::before{
+
+        &:last-child::before {
          content: '/';
          display: inline;
          margin-right: 0.3em;
@ -319,6 +402,7 @@ img {
    grid-column: start/end;
    grid-row: span 1;
    word-break: break-word;
+
    p {
      margin: 0 0 1em 0;

@ -330,15 +414,15 @@ img {
  }

  @media (max-width: 640px) {
-    .display-name{
+    .display-name {
      grid-column-start: start;
    }
-    .address{
+    .address {
      grid-column-start: start;
      margin-top: -1em;
    }
-    .avatar{
-      grid-row-start: badges ;
+    .avatar {
+      grid-row-start: badges;
    }
  }
 }
@ -352,9 +436,10 @@ img {
  }
 }

-footer{
+footer {
  color: var(--front-fg-color);
-  a{
+
+  a {
    color: inherit;
  }
 }
--- a/application/src/types/StatsResponse.ts
+++ b/application/src/types/StatsResponse.ts
@ -0,0 +1,16 @@
+import { z } from 'zod'
+
+export const statsResponseSoftwareSchema = z.object({
+  name: z.string().nullable(),
+  nodeCount: z.number().int().min(0),
+  accountCount: z.number().int().min(0),
+  channelCount: z.number().int().min(0),
+  newNodeCount: z.number().int().min(0)
+})
+
+export const statsResponseSchema = z.object({
+  softwares: z.array(statsResponseSoftwareSchema)
+})
+
+export type StatsResponse = z.infer<typeof statsResponseSchema>
+export type StatsResponseSoftware = z.infer<typeof statsResponseSoftwareSchema>