Add scribe.privacyredirect.com

Unknown if this will actually stop them, but at least I can show my intent. User agents sourced from https://darkvisitors.com/

.crystal-version

@@ -1 +1 @@
-1.5.0
+1.8.1

CHANGELOG

@@ -1,3 +1,41 @@
+Unreleased
+
+* Add a bunch of well-known, LLM scrapers to robots.txt
+* Add command to tag releases
+* Modernize nix config
+* Added scribe.manasiwibi.com instance
+
+2023-12-18
+
+* Added release script
+* Update License to include https and package.json
+* Add Nix package and NixOS moodule
+* Remove outdated postgres check and dep from shell.nix
+* Fixed Dockerfile
+* Update Dockerfile to use multi-arch base image
+* Add Docker.arm64 file
+* Remove instances that promote hate
+
+2023-05-21
+
+* Remove unused carbon shard which should fix build failures
+
+2023-05-06
+
+* Upgrade to Lucky framework 1.0.0
+* Upgrade to Crystal version 1.8.1
+* If embedded media has a caption, it will now be displayed
+
+2023-03-25
+
+* Headings now have an ID so readers can link to a part if an article
+* If a URL contains `global-identity-2`, Scribe will now correctly parse the article ID.
+
+2022-11-06
+
+* Fix viewing articles if the URL has a trailing slash
+* Update to nodejs 16.18.0
+
 2022-10-30
 
 * Update to nodejs 16.18.0

Dockerfile

@@ -3,13 +3,13 @@ WORKDIR /tmp_build
 
 COPY package.json .
 COPY yarn.lock .
-RUN yarn install --no-progress --frozen-lockfile
+RUN yarn install --network-timeout 120000 --no-progress --frozen-lockfile
 
 COPY webpack.mix.js .
 COPY src ./src
 RUN yarn prod
 
-FROM crystallang/crystal:1.5-alpine as lucky_build
+FROM 84codes/crystal:1.8.1-alpine as lucky_build
 ENV SKIP_LUCKY_TASK_PRECOMPILATION="1"
 RUN apk add yaml-static
 WORKDIR /tmp_build

LICENSE

@@ -1,7 +1,7 @@
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -643,7 +643,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -658,4 +658,4 @@ specific requirements.
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU AGPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.

default.nix

@@ -0,0 +1,58 @@
+{ crystal
+, mkYarnPackage
+, fetchYarnDeps
+}:
+
+let
+  version = "1.0.0";
+
+  ui = mkYarnPackage {
+    pname = "scribe-ui";
+    inherit version;
+    src = ./.;
+    packageJSON = ./package.json;
+
+    offlineCache = fetchYarnDeps {
+      yarnLock = ./yarn.lock;
+      sha256 = "sha256-ixnGRTTKq20tnjOnHeibu12a+n3edV1eM5Om2iNO9fo=";
+    };
+
+    configurePhase = ''
+      runHook preConfigure
+      cp -r $node_modules node_modules
+      chmod +w node_modules
+      runHook postConfigure
+    '';
+
+    buildPhase = ''
+      runHook preBuild
+      export HOME=$(mktemp -d)
+      OUTPUT_DIR=$out yarn --offline prod
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mkdir -p "$out"
+      mv public "$out/public"
+    '';
+    distPhase = "true";
+  };
+in
+crystal.buildCrystalPackage rec {
+  pname = "scribe";
+  inherit version;
+
+  src = ./.;
+  shardsFile = ./shards.nix;
+
+  preBuild = ''
+    cp -a ${ui}/public/mix-manifest.json public/mix-manifest.json
+  '';
+
+  doCheck = false;
+  doInstallCheck = false;
+  format = "shards";
+  postInstall = ''
+    cp -r ${ui}/public "$out/public"
+  '';
+}

docs/instances.json

@@ -4,8 +4,11 @@
   "https://scribe.citizen4.eu",
   "https://scribe.bus-hit.me",
   "https://scribe.froth.zone",
-  "https://scribe.esmailelbob.xyz",
   "https://scribe.privacydev.net",
   "https://scribe.rawbit.ninja",
-  "https://sc.vern.cc"
+  "https://sc.vern.cc",
+  "https://m.opnxng.com",
+  "https://scribe.manasiwibi.com",
+  "https://scribe.r4fo.com",
+  "https://scribe.privacyredirect.com"
 ]

docs/instances.md

@@ -5,12 +5,16 @@
 * <https://scribe.citizen4.eu>
 * <https://scribe.bus-hit.me>
 * <https://scribe.froth.zone>
-* <https://scribe.esmailelbob.xyz>
 * <https://scribe.privacydev.net>
 * <https://scribe.rawbit.ninja>
-* <http://scribe.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion> (Tor)
+* <https://m.opnxng.com>
+* <https://scribe.manasiwibi.com>
+* <https://scribe.r4fo.com>
+* <https://scribe.privacyredirect.com>
 * <http://w7uhv5lxhgck72hhimdglmusc54t4m6bionlmd5mvyddq3bs53mohqid.onion> (Tor)
 * <http://scribe.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion> (Tor)
+* <http://umxccfmp4gyfllsdlzkrnhpd3lxlf4necjolrz22yzcrgwflbrzgtiad.onion> (Tor)
+* <http://scribe.r4focoma7gu2zdwwcjjad47ysxt634lg73sxmdbkdozanwqslho5ohyd.onion> (Tor)
 * [sc.vern.i2p](http://vern3whzyfmjclq6snhlupma6nrmojghwp37tydfgqotj7sc6izq.b32.i2p) (I2P)
 
 ## How do I get my instance on this list?

flake.lock

@@ -17,15 +17,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1634282420,
-        "narHash": "sha256-YOI78SSF4Q/ZFoEgfO8Xy3EnjCW/F9VgB2Qz9YljzhI=",
+        "lastModified": 1701253981,
+        "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "0a68ef410b40f49de76aecb5c8b5cc5111bac91d",
+        "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
+        "ref": "nixos-unstable",
         "type": "indirect"
       }
     },

flake.nix

@@ -1,8 +1,21 @@
 {
-  inputs = { flake-utils.url = "github:numtide/flake-utils"; };
+  description = "Scribe";
+
+  inputs = {
+    nixpkgs.url = "nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
 
   outputs = { self, nixpkgs, flake-utils }:
-    flake-utils.lib.eachDefaultSystem (system:
-      let pkgs = nixpkgs.legacyPackages.${system};
-      in { devShell = import ./shell.nix { inherit pkgs; }; });
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let pkgs = nixpkgs.legacyPackages.${system};
+        in
+        {
+          devShell = import ./shell.nix { inherit pkgs; };
+          packages.default = pkgs.callPackage ./default.nix { };
+        })
+    // {
+      nixosModules.default = import ./module.nix self;
+    };
 }

module.nix

@@ -0,0 +1,111 @@
+self: { config, lib, pkgs, ... }:
+let
+  cfg = config.services.scribe;
+in
+{
+  options.services.scribe = {
+    enable = lib.mkEnableOption (lib.mdDoc "Enable or disable the Scribe service");
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = self.packages."${pkgs.system}".default;
+      description = lib.mdDoc "Overridable attribute of the scribe package to use.";
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "scribe";
+      description = lib.mdDoc "User to run the Scribe service as.";
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "scribe";
+      description = lib.mdDoc "Group to run the Scribe service as.";
+    };
+
+    appDomain = lib.mkOption {
+      type = lib.types.str;
+      description = lib.mdDoc ''
+        The domain that Scribe is being run on. This will appear on the Scribe homepage.
+      '';
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      description = lib.mdDoc "Port for the Scribe service to use.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.str;
+      description = lib.mdDoc ''
+        The path to a file containing environment varible to be set in Scribes environment.
+        This should be user to set SECRET_KEY_BASE, GITHUB_USERNAME, and GITHUB_PERSONAL_ACCESS_TOKEN. 
+        Descriptions of these settings can be found
+        [in the official docs](https://sr.ht/~edwardloveall/Scribe/#configuration).
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.scribe = {
+      description = "Scribe";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      environment = {
+        LUCKY_ENV = "production";
+        APP_DOMAIN = cfg.appDomain;
+        PORT = (toString cfg.port);
+      };
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/scribe";
+        EnvironmentFile = cfg.environmentFile;
+        Restart = "on-failure";
+        User = cfg.user;
+        Group = cfg.group;
+        UMask = "0007";
+        ProtectSystem = "strict";
+        ProtectClock = true;
+        ProtectKernelLogs = true;
+        SystemCallArchitectures = "native";
+        ProtectHome = true;
+        ProtectProc = "noaccess";
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        CapabilityBoundingSet = [
+          "~CAP_SYS_PTRACE"
+          "~CAP_SYS_ADMIN"
+          "~CAP_SETGID"
+          "~CAP_SETUID"
+          "~CAP_SETPCAP"
+          "~CAP_SYS_TIME"
+          "~CAP_KILL"
+          "~CAP_SYS_PACCT"
+          "~CAP_SYS_TTY_CONFIG "
+          "~CAP_SYS_CHROOT"
+          "~CAP_SYS_BOOT"
+          "~CAP_NET_ADMIN"
+        ];
+      };
+    };
+    users.users = lib.optionalAttrs (cfg.user == "scribe") {
+      "scribe" = {
+        group = "scribe";
+        isSystemUser = true;
+      };
+    };
+    users.groups = lib.optionalAttrs (cfg.group == "scribe") {
+      "scribe" = { };
+    };
+  };
+}

package.json

@@ -1,10 +1,11 @@
 {
-  "license": "UNLICENSED",
+  "name": "scribe-ui",
+  "license": "AGPL-3.0-or-later",
   "private": true,
   "dependencies": {
     "@rails/ujs": "^6.0.0",
     "compression-webpack-plugin": "^8.0.1",
-    "laravel-mix": "^6.0.28",
+    "laravel-mix": "^6.0.49",
     "modern-normalize": "^1.1.0",
     "postcss": "^8.3.6",
     "tufte-css": "^1.8.0",
@@ -22,5 +23,6 @@
     "resolve-url-loader": "^3.1.1",
     "sass": "^1.26.10",
     "sass-loader": "^10.0.2"
-  }
+  },
+  "version": "0.0.0"
 }

public/robots.txt

@@ -1,4 +1,55 @@
-# Learn more about robots.txt: https://www.robotstxt.org/robotstxt.html
-User-agent: *
-# 'Disallow' with an empty value allows all paths to be crawled
-Disallow:
+# ChatGPT-User
+User-agent: ChatGPT-User
+Disallow: /
+
+# cohere-ai
+User-agent: cohere-ai
+Disallow: /
+
+# anthropic-ai
+User-agent: anthropic-ai
+Disallow: /
+
+# Bytespider
+User-agent: Bytespider
+Disallow: /
+
+# CCBot
+User-agent: CCBot
+Disallow: /
+
+# FacebookBot
+User-agent: FacebookBot
+Disallow: /
+
+# Google-Extended
+User-agent: Google-Extended
+Disallow: /
+
+# GPTBot
+User-agent: GPTBot
+Disallow: /
+
+# omgili
+User-agent: omgili
+Disallow: /
+
+# Amazonbot
+User-agent: Amazonbot
+Disallow: /
+
+# Applebot
+User-agent: Applebot
+Disallow: /
+
+# PerplexityBot
+User-agent: PerplexityBot
+Disallow: /
+
+# PerplexityBot
+User-agent: PerplexityBot
+Disallow: /
+
+# YouBot
+User-agent: YouBot
+Disallow: /

script/build_static

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Exit if any subcommand fails
+set -e
+set -o pipefail
+
+todays_date=$(date "+%Y-%m-%d")
+if ! git show HEAD:src/version.cr | rg -q $todays_date; then
+  echo "Date in committed src/version.cr is not today's date ($todays_date)"
+  echo "Make sure that the file is both up to date and commited to git."
+  echo
+  echo "## src/version.cr"
+  git show HEAD:src/version.cr
+  # exit 1
+fi
+
+# Via: https://crystal-lang.org/reference/1.10/guides/static_linking.html#linux
+
+~/.docker/bin/docker run --rm -it -v $(pwd):/workspace -w /workspace crystallang/crystal:latest-alpine \
+  crystal build src/start_server.cr -o ubuntu_server --static --release

script/release

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+todays_date=$(date "+%Y-%m-%d")
+sed -i '' -E "s/[0-9]{4}-[0-9]{2}-[0-9]{2}/$todays_date/" src/version.cr
+
+# Delete the tag if it exists. This will show an error but it's fine
+git tag -d "$todays_date" || true
+
+echo
+echo "Bumped version to $todays_date"
+echo "Here are the commits since the last tag"
+echo "Update the Changelog"
+echo
+
+git log $(git describe --tags --abbrev=0)~..HEAD --oneline
+git tag "$todays_date"

script/system_check

@@ -17,17 +17,6 @@ if command_not_found "yarn"; then
   print_error "Yarn is not installed\n  See https://yarnpkg.com/lang/en/docs/install/ for install instructions."
 fi
 
-if command_not_found "createdb"; then
-  MSG="Please install the postgres CLI tools, then try again."
-  if is_mac; then
-    MSG="$MSG\nIf you're using Postgres.app, see https://postgresapp.com/documentation/cli-tools.html."
-  fi
-  MSG="$MSG\nSee https://www.postgresql.org/docs/current/tutorial-install.html for install instructions."
-
-  print_error "$MSG"
-fi
-
-
 ## CUSTOM PRE-BOOT CHECKS ##
 # example:
 # if command_not_running "redis-cli ping"; then

shard.lock

@@ -2,31 +2,27 @@ version: 2.0
 shards:
   authentic:
     git: https://github.com/luckyframework/authentic.git
-    version: 0.8.2
+    version: 1.0.0
 
   avram:
     git: https://github.com/luckyframework/avram.git
-    version: 0.23.0
+    version: 1.0.0
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
-    version: 1.2.1
+    version: 1.2.2
 
   cadmium_transliterator:
     git: https://github.com/cadmiumcr/transliterator.git
     version: 0.1.0+git.commit.46c4c14594057dbcfaf27e7e7c8c164d3f0ce3f1
 
-  carbon:
-    git: https://github.com/luckyframework/carbon.git
-    version: 0.2.1
-
   cry:
     git: https://github.com/luckyframework/cry.git
     version: 0.4.3
 
   crystar:
     git: https://github.com/naqvis/crystar.git
-    version: 0.2.0
+    version: 0.2.0+git.commit.56db8bb9dfbd5ed6d7908353405a5fae632a6561
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
@@ -38,15 +34,23 @@ shards:
 
   exception_page:
     git: https://github.com/crystal-loot/exception_page.git
-    version: 0.2.2
+    version: 0.3.0
+
+  fnv:
+    git: https://github.com/naqvis/crystal-fnv.git
+    version: 0.1.3
 
   habitat:
     git: https://github.com/luckyframework/habitat.git
     version: 0.4.7
 
+  html5:
+    git: https://github.com/naqvis/crystal-html5.git
+    version: 0.4.0
+
   lucky:
     git: https://github.com/luckyframework/lucky.git
-    version: 0.30.1
+    version: 1.0.0
 
   lucky_cache:
     git: https://github.com/luckyframework/lucky_cache.git
@@ -58,11 +62,11 @@ shards:
 
   lucky_flow:
     git: https://github.com/luckyframework/lucky_flow.git
-    version: 0.7.3
+    version: 0.9.0
 
   lucky_router:
     git: https://github.com/luckyframework/lucky_router.git
-    version: 0.5.1
+    version: 0.5.2
 
   lucky_task:
     git: https://github.com/luckyframework/lucky_task.git
@@ -82,7 +86,7 @@ shards:
 
   selenium:
     git: https://github.com/matthewmcgarvey/selenium.cr.git
-    version: 0.9.1
+    version: 0.10.0
 
   shell-table:
     git: https://github.com/luckyframework/shell-table.cr.git
@@ -98,9 +102,17 @@ shards:
 
   webdrivers:
     git: https://github.com/matthewmcgarvey/webdrivers.cr.git
-    version: 0.4.0
+    version: 0.4.1
+
+  webless:
+    git: https://github.com/matthewmcgarvey/webless.git
+    version: 0.1.0
 
   wordsmith:
     git: https://github.com/luckyframework/wordsmith.git
-    version: 0.3.0
+    version: 0.4.0
+
+  xpath2:
+    git: https://github.com/naqvis/crystal-xpath2.git
+    version: 0.1.3
 

shard.yml

@@ -8,18 +8,18 @@ targets:
   scribe:
     main: src/scribe.cr
 
-crystal: 1.5.0
+crystal: 1.8.1
 
 dependencies:
   lucky:
     github: luckyframework/lucky
-    version: ~> 0.30.1
+    version: ~> 1.0.0
+  avram:
+    github: luckyframework/avram
+    version: ~> 1.0.0
   authentic:
     github: luckyframework/authentic
-    version: ~> 0.8.2
-  carbon:
-    github: luckyframework/carbon
-    version: ~> 0.2.0
+    version: ~> 1.0.0
   lucky_env:
     github: luckyframework/lucky_env
     version: ~> 0.1.4
@@ -31,4 +31,4 @@ dependencies:
 development_dependencies:
   lucky_flow:
     github: luckyframework/lucky_flow
-    version: ~> 0.7.3
+    version: ~> 0.9.0

shards.nix

@@ -0,0 +1,176 @@
+{
+  authentic = {
+    owner = "luckyframework";
+    repo = "authentic";
+    rev = "v1.0.0";
+    sha256 = "0mc7xqh0zm4jg8vc1awlzr249fviiy1y40w4fvyvq959hlpd6zx4";
+  };
+  avram = {
+    owner = "luckyframework";
+    repo = "avram";
+    rev = "v1.0.0";
+    sha256 = "18w90m5iq0jy026zma05swh2am936j132fs3j730lq7x5yr8289c";
+  };
+  backtracer = {
+    owner = "sija";
+    repo = "backtracer.cr";
+    rev = "v1.2.2";
+    sha256 = "1rknyylsi14m7i77x7c3138wdw27i4f6sd78m3srw851p47bwr20";
+  };
+  cadmium_transliterator = {
+    owner = "cadmiumcr";
+    repo = "transliterator";
+    rev = "46c4c14594057dbcfaf27e7e7c8c164d3f0ce3f1";
+    sha256 = "15x9xbgybqrmqb7s5cpx3fgwysp5ld97vlvz8b196lqmyqnnp3d3";
+  };
+  cry = {
+    owner = "luckyframework";
+    repo = "cry";
+    rev = "v0.4.3";
+    sha256 = "0bcvpbi418855cq1jq911dv6r9wmg81rcvcirqrbw8fv2a093ss5";
+  };
+  crystar = {
+    owner = "naqvis";
+    repo = "crystar";
+    rev = "56db8bb9dfbd5ed6d7908353405a5fae632a6561";
+    sha256 = "0bzq7im3z3asr22wzwyj1z0m3m5aq5hh1kscp5gd8vjw192w2z2a";
+  };
+  db = {
+    owner = "crystal-lang";
+    repo = "crystal-db";
+    rev = "v0.11.0";
+    sha256 = "1ylfhpn64p72ywi39niqb179f61z08q4qd4hhjza05z18mdaghl3";
+  };
+  dexter = {
+    owner = "luckyframework";
+    repo = "dexter";
+    rev = "v0.3.4";
+    sha256 = "08fv3ns0wxkyr2rcifj3ihyaf7g4lsmfamfhdxbkdkmxa9l1z6cj";
+  };
+  exception_page = {
+    owner = "crystal-loot";
+    repo = "exception_page";
+    rev = "v0.3.0";
+    sha256 = "1w82283mgaaw1hy5xk997a1av4sxaa01ydipbxm5nb9nq7fgfydk";
+  };
+  fnv = {
+    owner = "naqvis";
+    repo = "crystal-fnv";
+    rev = "v0.1.3";
+    sha256 = "1vhy3j0ifc0rlrx5b6wbpcvjzw15k303jrz3bzvnxqvi600fvv2b";
+  };
+  habitat = {
+    owner = "luckyframework";
+    repo = "habitat";
+    rev = "v0.4.7";
+    sha256 = "0d183pnswgjwqg388zmnx7s41ai88ca96nl5cybi0z6icr5npw64";
+  };
+  html5 = {
+    owner = "naqvis";
+    repo = "crystal-html5";
+    rev = "v0.4.0";
+    sha256 = "0mr4vd4bl3a22jl8h698zrh8rz6m5lm2lcyx11055gn6fw0yq57k";
+  };
+  lucky = {
+    owner = "luckyframework";
+    repo = "lucky";
+    rev = "v1.0.0";
+    sha256 = "13by6bbgpbbbdncgj87cqy5y6z7s9zb3nr88dh3fwl5mfgygk66z";
+  };
+  lucky_cache = {
+    owner = "luckyframework";
+    repo = "lucky_cache";
+    rev = "v0.1.1";
+    sha256 = "1ic9nfmiv89q5v82ybshd9xqnwv62bv8a5n8rhmsm9cwvdhgc92x";
+  };
+  lucky_env = {
+    owner = "luckyframework";
+    repo = "lucky_env";
+    rev = "v0.1.4";
+    sha256 = "0rcz0kh9rkypgm34r7maqqmgirxblhwzycwxpp0y9ai68lq71qxk";
+  };
+  lucky_flow = {
+    owner = "luckyframework";
+    repo = "lucky_flow";
+    rev = "v0.9.0";
+    sha256 = "1gyxba7lbjhzbd7a5hcswr3i04mz6rqypihhpgx213aa2685c0mw";
+  };
+  lucky_router = {
+    owner = "luckyframework";
+    repo = "lucky_router";
+    rev = "v0.5.2";
+    sha256 = "1gl93rijnbaqybpry19rn951kbx1q1bb5w0npdp4fm0r212b3yh8";
+  };
+  lucky_task = {
+    owner = "luckyframework";
+    repo = "lucky_task";
+    rev = "v0.1.1";
+    sha256 = "0w0rnf22pvj3lp5z8c4sshzwhqgwpbjpm7nry9mf0iz3fa0v48f7";
+  };
+  monads = {
+    owner = "alex-lairan";
+    repo = "monads";
+    rev = "v1.0.0";
+    sha256 = "0wwhsmnzsmw03dn2j4n75sprp4baxg24i1hn1xhfzz9b33rmlxxf";
+  };
+  pg = {
+    owner = "will";
+    repo = "crystal-pg";
+    rev = "v0.26.0";
+    sha256 = "04fwbgrlf2nzma0p2c8ki7p8sk113jhziq2al3ivif2lpmhr39fy";
+  };
+  pulsar = {
+    owner = "luckyframework";
+    repo = "pulsar";
+    rev = "v0.2.3";
+    sha256 = "03pp0r1klqk49fkzjwg9mnxqplv6pdfjn6a1p59f2w1ha5piyy90";
+  };
+  selenium = {
+    owner = "matthewmcgarvey";
+    repo = "selenium.cr";
+    rev = "v0.10.0";
+    sha256 = "062baqafz2rn9czaj8wl2b1l7ngxdph2j8xcr088f2kd8bb0hj7v";
+  };
+  shell-table = {
+    owner = "luckyframework";
+    repo = "shell-table.cr";
+    rev = "v0.9.3";
+    sha256 = "046vymm2r37c6j5bqyjzxdgg5h62slsannzvfhbckkv2r9chwd3w";
+  };
+  splay_tree_map = {
+    owner = "wyhaines";
+    repo = "splay_tree_map.cr";
+    rev = "v0.2.2";
+    sha256 = "0196zpg0v190dv23mwnbia35znxz2j2g8dqynd2b8827qiwmz1vn";
+  };
+  teeplate = {
+    owner = "luckyframework";
+    repo = "teeplate";
+    rev = "v0.8.5";
+    sha256 = "1kr05qrp674rph1324wry57gzvgvcvlz0w27brlvdgd3gi4s8sdj";
+  };
+  webdrivers = {
+    owner = "matthewmcgarvey";
+    repo = "webdrivers.cr";
+    rev = "v0.4.1";
+    sha256 = "05q6z1rv29hrwq77wpas2ki4alwhx4fpallb94q4m9g5h5vfn6ag";
+  };
+  webless = {
+    owner = "matthewmcgarvey";
+    repo = "webless";
+    rev = "v0.1.0";
+    sha256 = "0fg79wy3fq0af77jm121pqfm43dzb7l5rlx13vrl74pgqagms0ih";
+  };
+  wordsmith = {
+    owner = "luckyframework";
+    repo = "wordsmith";
+    rev = "v0.4.0";
+    sha256 = "13fsmwdh431smbmsv869pa8p34g1hqd84za33xsymsycq5459xq2";
+  };
+  xpath2 = {
+    owner = "naqvis";
+    repo = "crystal-xpath2";
+    rev = "v0.1.3";
+    sha256 = "17jl0br2fibc22sz9qdpsqd17rsmnar0jwh4iq25y8rg64pgb1h0";
+  };
+}

shell.nix

@@ -1,17 +1,16 @@
 { pkgs ? import <nixpkgs> { } }:
 
 pkgs.mkShell {
-  shellHook = ''
-    export PKG_CONFIG_PATH=${pkgs.openssl.dev}/lib/pkgconfig
-  '';
   buildInputs = with pkgs; [
     crystal
     lucky-cli
     overmind
     nodejs
-    openssl.dev
-    postgresql
+    openssl
+    pkg-config
     shards
     yarn
+    crystal2nix
+    pcre
   ];
 }

spec/classes/article_id_parser_spec.cr

@@ -78,6 +78,14 @@ describe ArticleIdParser do
     result.should eq(Monads::Just.new("888888abcdef"))
   end
 
+  it "parses the post id for global identity 2 redirects" do
+    request = resource_request("/m/global-identity-2?redirectUrl=https%3A%2F%2Fexample.com%2Fmy-post-999999abcdef")
+
+    result = ArticleIdParser.parse(request)
+
+    result.should eq(Monads::Just.new("999999abcdef"))
+  end
+
   it "returns Nothing if path is a username" do
     request = resource_request("/@ba5eba11")
 

spec/classes/embedded_converter_spec.cr

@@ -37,6 +37,44 @@ describe EmbeddedConverter do
         )
       )
     end
+
+    context "and a caption exists" do
+      it "returns an EmbeddedContent node with caption" do
+        store = GistStore.new
+        paragraph = PostResponse::Paragraph.from_json <<-JSON
+          {
+            "name": "ab12",
+            "text": "Caption",
+            "type": "IFRAME",
+            "href": null,
+            "layout": "INSET_CENTER",
+            "markups": [],
+            "iframe": {
+              "mediaResource": {
+                "id": "abc123",
+                "href": "https://twitter.com/user/status/1",
+                "iframeSrc": "https://cdn.embedly.com/widgets/...",
+                "iframeWidth": 500,
+                "iframeHeight": 281
+              }
+            },
+            "metadata": null
+          }
+        JSON
+        caption = FigureCaption.new(children: [Text.new("Caption")] of Child)
+
+        result = EmbeddedConverter.convert(paragraph, store)
+
+        result.should eq(
+          EmbeddedContent.new(
+            src: "https://cdn.embedly.com/widgets/...",
+            originalWidth: 500,
+            originalHeight: 281,
+            caption: caption,
+          )
+        )
+      end
+    end
   end
 
   context "when the mediaResource has a blank iframeSrc value" do

spec/components/page_content_spec.cr

@@ -249,6 +249,7 @@ describe PageContent do
   end
 
   it "renders embedded content" do
+    caption_children = [Text.new("Caption")] of Child
     page = Page.new(
       title: "Title",
       author: user_anchor_factory,
@@ -258,6 +259,7 @@ describe PageContent do
           src: "https://example.com",
           originalWidth: 1000,
           originalHeight: 600,
+          caption: FigureCaption.new(children: caption_children)
         ),
       ] of Child
     )
@@ -268,6 +270,11 @@ describe PageContent do
       <figure>
         <iframe src="https://example.com" width="800" height="480" frameborder="0" allowfullscreen="true">
         </iframe>
+        <label class="margin-toggle" for="#{caption_children.hash}">&#9997;&#xFE0E;</label>
+        <input class="margin-toggle" type="checkbox" id="#{caption_children.hash}">
+        <span class="marginnote">
+          Caption
+        </span>
       </figure>
     HTML
   end

spec/setup/configure_lucky_flow.cr

@@ -4,10 +4,10 @@
 LuckyFlow.configure do |settings|
   settings.stop_retrying_after = 200.milliseconds
   settings.base_uri = Lucky::RouteHelper.settings.base_uri
-
+  
   # By default, LuckyFlow is set in "headless" mode (no browser window shown).
   # Uncomment this to enable running `LuckyFlow` in a Google Chrome window instead.
   # Be sure to disable for CI.
   # settings.driver = LuckyFlow::Drivers::Chrome
 end
-Spec.before_each { LuckyFlow::Server::INSTANCE.reset }
+LuckyFlow::Spec.setup

spec/setup/reset_emails.cr

@@ -1,3 +0,0 @@
-Spec.before_each do
-  Carbon::DevAdapter.reset
-end

spec/spec_helper.cr

@@ -2,13 +2,14 @@ ENV["LUCKY_ENV"] = "test"
 ENV["DEV_PORT"] = "5001"
 require "spec"
 require "lucky_flow"
+require "lucky_flow/ext/lucky"
+require "lucky_flow/ext/avram"
 require "../src/app"
 require "./support/flows/base_flow"
 require "./support/**"
 require "../db/migrations/**"
 require "./setup/**"
 
-include Carbon::Expectations
 include Lucky::RequestExpectations
 include LuckyFlow::Expectations
 

src/classes/article_id_parser.cr

@@ -10,7 +10,7 @@ class ArticleIdParser
   def parse(request : HTTP::Request) : Maybe
     from_params = post_id_from_params(request.query_params)
     from_path = post_id_from_path(request.path)
-    from_path.or(from_params)
+    from_params.or(from_path)
   end
 
   private def post_id_from_path(request_path : String)

src/classes/embedded_converter.cr

@@ -34,11 +34,19 @@ class EmbeddedConverter
       EmbeddedContent.new(
         src: media.iframeSrc,
         originalWidth: media.iframeWidth,
-        originalHeight: media.iframeHeight
+        originalHeight: media.iframeHeight,
+        caption: caption
       )
     end
   end
 
+  private def caption : FigureCaption?
+    if !paragraph.text.blank?
+      children = [Text.new(paragraph.text || "")] of Child
+      FigureCaption.new(children: children)
+    end
+  end
+
   private def custom_embed(media : PostResponse::MediaResource) : Embedded
     if media.href.starts_with?(GIST_HOST_AND_SCHEME)
       GithubGist.new(href: media.href, gist_store: gist_store)

src/components/page_content.cr

@@ -40,6 +40,9 @@ class PageContent < BaseComponent
         frameborder: "0",
         allowfullscreen: true,
       )
+      if caption = child.caption
+        render_child(caption)
+      end
     end
   end
 

src/models/nodes.cr

@@ -147,8 +147,14 @@ module Nodes
     MAX_WIDTH = 800
 
     getter src : String
+    getter caption : FigureCaption?
 
-    def initialize(@src : String, @originalWidth : Int32, @originalHeight : Int32)
+    def initialize(
+      @src : String,
+      @originalWidth : Int32,
+      @originalHeight : Int32,
+      @caption : FigureCaption? = nil
+    )
     end
 
     def width
@@ -168,7 +174,10 @@ module Nodes
     end
 
     def ==(other : EmbeddedContent)
-      other.src == src && other.width == width && other.height == height
+      other.src == src &&
+        other.width == width &&
+        other.height == height &&
+        other.caption == caption
     end
 
     def empty?

src/shards.cr

@@ -5,6 +5,5 @@ LuckyEnv.load?(".env")
 # Require your shards here
 require "avram"
 require "lucky"
-require "carbon"
 require "authentic"
 require "monads"

src/version.cr

@@ -1,3 +1,3 @@
 module Scribe
-  VERSION = "2023-03-25"
+  VERSION = "2023-12-18"
 end

tasks.cr

@@ -7,5 +7,6 @@ require "lucky_task"
 require "./tasks/**"
 require "./db/migrations/**"
 require "lucky/tasks/**"
+require "avram/lucky/tasks"
 
 LuckyTask::Runner.run