Merge branch 'master' into 20210510-remove-v2-onion-creation.

README.md

@@ -18,6 +18,7 @@ NB: bugs should be reported through `Issues`, above.
 
 ### EOTK In the News
 
+* Apr 2021 [The Intercept launches onionsite using EOTK](https://theintercept.com/2021/04/28/tor-browser-onion/)
 * Oct 2020 [Brave browser launches onionsite using EOTK](https://brave.com/new-onion-service/)
 * Oct 2019 [BBC News launches 'dark web' Tor mirror](https://www.bbc.co.uk/news/technology-50150981)
 * Oct 2019 [BBC launches dark web news site in bid to dodge censors](https://www.cityam.com/bbc-launches-dark-web-news-site-in-bid-to-dodge-censors/)

demo.d/example.tconf

@@ -35,7 +35,15 @@
 # set x_from_onion_value 1
 
 # When you're proving SSL ownership, you may want arbitrary text
-# strings to be returned for a GET upon an arbitrary "/path" regexp
+# strings to be returned for a GET upon an arbitrary "/path"
+#
+# set ssl_proof_csv \
+#     /.well_known/fookey1,fooval1 \
+#     /.well_known/fookey2,fooval2
+
+# ...and a similar, more generic, regular-expression-based solution
+# for fixed strings to be returned for a GET upon an arbitrary
+# location (restricted to HTTPS-only)
 #
 # set hardcoded_endpoint_csv ^/regexp/pattern/?$,stringvalue ...
 

docs.d/HOW-TO-INSTALL.md

@@ -97,7 +97,7 @@ that you will need.
 You can then add `set ssl_mkcert 1` to configurations, and your
 `mkcert` root certificate will be used to sign the resulting onion
 certificates. You can [install that certificate into your local copy
-of Tor Browser](docs.d/ADDING-A-ROOT-CERTIFICATE-TO-TOR-BROWSER.md);
+of Tor Browser](/docs.d/ADDING-A-ROOT-CERTIFICATE-TO-TOR-BROWSER.md);
 of course it will not work for anyone else.
 
 ## Visit `/hello-onion/` URLs
@@ -119,27 +119,38 @@ rendering these issues moot.
 
 # Proving Your Ownership To A Certificate Authority / Hardcoded Content
 
-## IMPORTANT: if all of your "proof" URLs have DIFFERENT pathnames?
+## IMPORTANT: if your "proof" URLs have DIFFERENT pathnames?
 
-Small amounts of plain-text page content may be embedded using
-regular-expressions for pathnames; this is done using
-`hardcoded_endpoint_csv` and the following example will emit
-`FOOPROOF` (or `BARPROOF`) for accesses to `/www/.well_known/foo` or
-`../.well_known/bar` respectively, ignoring trailing slashes.  Note
-the use of double-backslash to escape "dots" in the regular
-expression, and use of backslash-indent to continue/enable several
-such paths.
+Small amounts of plain-text page content may be embedded using small,
+fixed pathname strings; this is done using `ssl_proof_csv` and the
+following example will emit `FOOPROOF` (or `BARPROOF`) for accesses to
+`/www/.well_known/foo` (or `.../bar`) respectively.
+
+Note: unlike the previous mechanism which was based on the
+regular-expression-based `hardcoded_endpoint_csv`, these strings are
+checked verbatim against the location, so that `/.well_known/FOO`
+becomes `location "/.well_known/FOO" {...}` in the NGINX
+configuration.
+
+Also, as an improvement to the previous mechanism, these endpoints are
+available in **both** HTTP and HTTPS, irrespective of the state of
+the `force_https` setting.
+
+Example code:
 
 ```
 # demo: CSV list to implement ownership proof URIs for EV SSL issuance
-set hardcoded_endpoint_csv \
-    ^/www/\\.well_known/foo/?$,"FOOPROOF" \
-    ^/www/\\.well_known/bar/?$,"BARPROOF"
+set ssl_proof_csv \
+    /.well_known/FOO,FOOPROOF \
+    /.well_known/BAR,BARPROOF
 ```
 
-## IMPORTANT: if all your "proof" URLs have THE SAME pathname?
+It is advisable to comment these lines out and reconfigure/reload your
+onions, after you obtain a certificate.
 
-The `hardcoded_endpoint_csv` hack works okay if all the proof URLs are
+## IMPORTANT: if your "proof" URLs have THE SAME pathname?
+
+The `ssl_proof_csv` hack works okay if all the proof URLs are
 different; but if Digicert (or whomever) give you the same pathname
 (e.g. `/.well-known/pki-validation/fileauth.txt`) for all of the
 onions, what do you do?
@@ -154,7 +165,7 @@ Answer: you use "splicing".  If you have onion addresses named
 customise as necessary:
 
 ```
-    location ~ "^/\\.well-known/pki-validation/fileauth\\.txt$" {
+    location = "/.well-known/pki-validation/fileauth.txt" {
       return 200 "RESPECTIVE-XXX-OR-YYY-PROOF-STRING-GOES-HERE";
     }
 ```

lib.d/do-configure.pl

@@ -619,6 +619,7 @@ my @set_blank = qw(
     referer_blacklist_re
     referer_whitelist
     referer_whitelist_re
+    ssl_proof_csv
     tor_intros_per_daemon
     user_agent_blacklist
     user_agent_blacklist_re

lib.d/lint.pl

@@ -129,6 +129,7 @@ my %known =
      'SOFTMAP_TOR_WORKERS' => 1,
      'SSL_DIR' => 1, # where ssl certs for the current project live
      'SSL_MKCERT' => 1,
+     'SSL_PROOF_CSV' => 1,
      'SSL_TOOL' => 1,
      'SUPPRESS_HEADER_CSP' => 1,
      'SUPPRESS_HEADER_HPKP' => 1,

templates.d/nginx.conf.txt

@@ -608,11 +608,25 @@ http {
     # tor2web not suppressed
     %%ENDIF
 
-    # tell the client to try again as HTTPS without ever leaving the onion
-    # use 307 / temporary redirect because your URIs may change in future
-    # use $host (not $server) to copy-over subdomains, etc, transparently
-    # SEND BACK ORIGINAL PARAMS, FIX THEM ONLY UPON FORWARD TO THE PROXY.
-    return 307 https://$host$request_uri;
+    location / {
+        # tell the client to try again as HTTPS without ever leaving the onion
+        # use 307 / temporary redirect because your URIs may change in future
+        # use $host (not $server) to copy-over subdomains, etc, transparently
+        # SEND BACK ORIGINAL PARAMS, FIX THEM ONLY UPON FORWARD TO THE PROXY.
+        return 307 https://$host$request_uri;
+    }
+
+    %%IF %SSL_PROOF_CSV%
+    # ssl_proof_csv: 1=fixed_path,2=response
+    %%CSV %SSL_PROOF_CSV%
+    location "%1%" {
+      return 200 "%2%";
+    }
+
+    %%ENDCSV
+    %%ELSE
+    # no ssl_proof_csv
+    %%ENDIF
   }
   %%ELSE
   # FORCE_HTTPS is not in use, cleartext data may traverse the internet
@@ -670,15 +684,28 @@ http {
     # no "hello-onion" endpoint
     %%ENDIF
 
-    %%IF %HARDCODED_ENDPOINT_CSV%
-    # hardcoded_endpoints: 1=path_re,2=response
-    %%CSV %HARDCODED_ENDPOINT_CSV%
-    location ~ "%1%" {
-      return 200 %2%;
+    %%IF %SSL_PROOF_CSV%
+    # ssl_proof_csv: 1=fixed_path,2=response
+    %%CSV %SSL_PROOF_CSV%
+    location "%1%" {
+      return 200 "%2%";
     }
+
     %%ENDCSV
     %%ELSE
-    # no hardcoded_endpoints
+    # no ssl_proof_csv
+    %%ENDIF
+
+    %%IF %HARDCODED_ENDPOINT_CSV%
+    # hardcoded_endpoint_csv: 1=path_re,2=response
+    %%CSV %HARDCODED_ENDPOINT_CSV%
+    location ~ "%1%" {
+      return 200 "%2%";
+    }
+
+    %%ENDCSV
+    %%ELSE
+    # no hardcoded_endpoint_csv
     %%ENDIF
 
     %%IF exists templates.d/nginx-site-%ONION_ADDRESS%.conf