add tinyweb-site theme: clean, light, minimal

174 tests covering URL normalization, FTS5 query sanitization, SSRF/CSRF
guards, sharing-mode logic, DB schema and upsert paths, handler
end-to-end flows, and gateway body-size / mesh-whitelist guards. Each
recent bug-fix commit (6ffd38d, 1bc695f, 8dffd8c) has an explicit
regression test in test_regressions.py. One xfail documents a minor
latent bug in clean_url where port 80 is not stripped from upgraded
https URLs.

.dockerignore

@@ -0,0 +1,15 @@
+__pycache__/
+**/__pycache__/
+*.pyc
+index.db*
+index.hnsw
+tinyweb_identity
+.git/
+.gitignore
+*.md
+.env
+.env.*
+.venv/
+venv/
+models/
+.DS_Store

.forgejo/workflows/build.yml

@@ -0,0 +1,81 @@
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: https://code.forgejo.org/actions/checkout@v4
+
+      - name: Set up Python
+        run: |
+          apt-get update && apt-get install -y python3 python3-pip python3-venv jq curl
+          curl -fsSL https://get.docker.com | sh
+          pip3 install --break-system-packages -r requirements.txt
+          pip3 install --break-system-packages pyinstaller
+
+      - name: Build with PyInstaller
+        run: |
+          pyinstaller --onefile --console --name TinyWeb app.py
+
+      - name: Prepare artifact
+        run: |
+          cp dist/TinyWeb TinyWeb-linux-x64
+          chmod +x TinyWeb-linux-x64
+          ls -la TinyWeb-linux-x64
+
+      - name: Get Release ID
+        if: startsWith(github.ref, 'refs/tags/v')
+        id: release
+        run: |
+          TAG="${{ github.ref_name }}"
+          REPO="${{ github.repository }}"
+          TOKEN="${{ secrets.FORGEJO_TOKEN }}"
+          RELEASE_JSON=$(curl -s "https://git.derickphan.com/api/v1/repos/$REPO/releases/tags/$TAG" \
+            -H "Authorization: token $TOKEN")
+          echo "$RELEASE_JSON"
+          RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
+          echo "release_id=$RELEASE_ID" >> $FORGEJO_OUTPUT
+
+      - name: Upload to Release
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          FILE=TinyWeb-linux-x64
+          RELEASE_ID="${{ steps.release.outputs.release_id }}"
+          REPO="${{ github.repository }}"
+          TOKEN="${{ secrets.FORGEJO_TOKEN }}"
+          curl -X POST "https://git.derickphan.com/api/v1/repos/$REPO/releases/$RELEASE_ID/assets" \
+            -H "Authorization: token $TOKEN" \
+            -F "attachment=@$FILE"
+
+      - name: Login to Registry
+        run: |
+          echo "${{ secrets.REGISTRY_TOKEN }}" | docker login registry.derickphan.com -u _ --password-stdin
+
+      - name: Build and push Docker image
+        run: |
+          TAG="${{ github.ref_name }}"
+          if [ -z "$TAG" ]; then
+            TAG="latest"
+          fi
+          # Configure Docker daemon with DNS
+          mkdir -p ~/.docker
+          cat > ~/.docker/daemon.json << 'EOF'
+          {
+            "dns": ["8.8.8.8", "1.1.1.1"],
+            "builder": {
+              "features": {
+                "buildkit": true
+              }
+            }
+          }
+          EOF
+          # Build with buildkit
+          DOCKER_BUILDKIT=1 docker build --network=host -t registry.derickphan.com/tinyweb:$TAG .
+          docker push registry.derickphan.com/tinyweb:$TAG
+

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+tinyweb_identity
+index.db
+index.db-shm
+index.db-wal
+models/
+index.hnsw

.runner

@@ -0,0 +1,11 @@
+{
+  "WARNING": "This file is automatically generated by forgejo-runner. Do not edit it manually unless you know what you are doing. Removing this file will cause act runner to re-register as a new runner.",
+  "id": 14,
+  "uuid": "2f3252bd-5f69-4d3c-8f77-8b54caf784af",
+  "name": "nixos-runner",
+  "token": "993ebcbb295c7c58ac35fd5c2ec07cc752c99152",
+  "address": "https://git.derickphan.com",
+  "labels": [
+    "ubuntu-latest:docker"
+  ]
+}

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install build tools for packages like hnswlib
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    g++ \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+RUN mkdir -p /data \
+    && ln -sf /data/index.db index.db \
+    && ln -sf /data/tinyweb_identity tinyweb_identity
+
+ENV PYTHONUNBUFFERED=1
+
+EXPOSE 8080
+
+ENTRYPOINT ["./entrypoint.sh"]

LICENSE

@@ -0,0 +1,574 @@
+GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License giving you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+are made publicly available of their being a derivative work, need not
+be distributed to others.
+
+  For example, if you modify a part of a free program, you are
+not required to distribute the object code for the modified version
+itself; however, the GNU Affero General Public License requires you
+to provide source code for any version of the program that you use
+or modify.  This requirement is similar to the requirement that
+the user can receive the source code if they distribute a copy.
+
+  Also, if you link or combine the program with any other software
+that contains code covered by this License (or any work based on the
+program), you must provide the source code for that combined work
+as well.  The GNU Affero General Public License normally requires
+that any work that you distribute or publish that in whole or in
+part contains or is derived from the program or any part thereof,
+to be licensed as a whole at no charge to all third parties under
+the terms of this License.  This is known as "providing source code"
+or "making available" the work.
+
+  An "aggregated" or "combined" work is not covered by this License
+if you do not meet these conditions, and you must provide the source
+code as above.  Additionally, aggregating works does not exempt you
+from the requirements of this License.
+
+  Specifically, if you make an "aggregate" or "combined" work by
+combining this program with other software (or any work based on this
+program) on a volume of a storage or distribution medium, you must
+provide the source code for the combined work as above.  This
+requirement is intended to ensure that any user of the combined work
+gets the source code that you made available, and can exercise the
+right to modify and re-distribute the combined work.
+
+  This License is specifically intended to limit any attempt to
+place your modifications under a license that would restrict re-use
+or further modification by others.  This is to ensure that any
+derivative work you create will be available under the same license
+as the original, so that any derivative work can be re-distributed
+under the same conditions as the original.
+
+  Finally, this License is not intended to limit your rights under
+fair use or other limitations on exclusive rights, such as patents
+or trademarks.  This License does not grant you any rights to the
+names of the authors or copyright holders, nor to trade names,
+trademarks, or service marks, except as needed for the normal and
+customary use in describing the origin of the work and reproducing
+the content of the notice file.
+
+  The source code for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided in copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must contain prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must contain prominent notices stating that it is
+    released under this License and any conditions added under section 7.
+    This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the legal rights of the compilation's users beyond
+what the individual works permit.  Inclusion of a covered work in an
+aggregate does not cause this License to apply to the other parts of
+the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in
+    accord with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A
+product is a consumer product regardless of whether the product has
+substantial commercial, industrial or non-consumer uses, unless such
+uses represent the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions applicable to the entire Program shall be treated
+as though they were included in this License, to the extent that they
+are valid under applicable law.  If additional permissions apply only to
+part of the Program, then that part may be used separately under those
+permissions, but the entire Program remains governed by this License
+without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as
+you received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, or selling the work,
+or by making, using, or selling the work.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available to anyone
+in the United States, you may not convey the work under this License.
+This is done by providing access to copy the corresponding source code
+from a network server at no charge.
+
+  If, during the execution of the Program, the Program is transmitted
+to a user or a computer, either the source code or object code, you
+must meet the requirements of this License regarding the
+Corresponding Source of the work.  You must make sure that the source
+code or object code (as applicable) is available for such users to
+copy and modify, and to run, for their own use, the corresponding
+source in accordance with this License.  This requirement applies
+both to the work as stand-alone and to the work as part of an
+aggregate.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user
+through a computer network, with no transfer of a copy, is not
+conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  12. No Warranty
+
+  THE PROGRAM IS PROVIDED WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK
+AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD
+THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY
+SERVICING, REPAIR OR CORRECTION.
+
+  13. Disclaimer of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR
+THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+  14. Interpretation of Sections 12 and 13.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+  The hypothetical commands `show w' and `show c' should show the appropriate
+  parts of the General Public License.  Of course, your program's commands
+  might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+  if any, to sign a "copyright disclaimer" for the program, if necessary.
+  For more information on this, and how to apply and follow the GNU AGPL,
+  see <https://www.gnu.org/licenses/>.

README.md

@@ -0,0 +1,259 @@
+# TinyWeb
+
+A personal, decentralized search engine built on the [Reticulum](https://reticulum.network/) mesh network. Curate your own index of web pages, search it locally, and share collections with friends over an encrypted mesh. No algorithms, no ads, no tracking.
+
+## Features
+
+- **Personal search index** — Save pages you find valuable, search them with full-text search (SQLite FTS5)
+- **Tagging** — Organize saved pages with comma-separated tags
+- **Bookmarklet** — One-click indexing from any browser tab
+- **Subscriptions** — Subscribe to friends' TinyWeb instances over Reticulum and search their indexes alongside yours
+- **Custom templates** — Full HTML/CSS/JS template editor to personalize your instance
+- **Import/export** — JSON-based backup and restore
+- **Mesh-native** — Works over Reticulum without the internet; encrypted and decentralized by default
+- **Forum plugin** — Optional link-sharing discussion board over the mesh (see Forum section below)
+
+## Performance & Scale
+
+### Search Speed
+
+| Pages indexed | Search speed | Notes |
+|--------------|-------------|-------|
+| 1,000 | ~50ms | Fast local FTS5 |
+| 10,000 | ~50-100ms | Full-text search |
+| 100,000 | ~100-200ms | Combined BM25 + semantic |
+| 500,000 | ~200-400ms | With semantic enabled |
+| 1,000,000 | ~300-500ms | Hybrid search |
+
+*Times are estimates for combined BM25 + semantic search. Actual performance varies by hardware, storage type (SSD/HDD), and search complexity.*
+
+### Concurrent Connections
+
+- Database pool: 16 simultaneous connections
+- Suitable for single-user + a few subscriptions
+
+### Export
+
+- Paginated at 10,000 pages per request
+- Use `?batch=N` to export in chunks: `/export?batch=0`, `/export?batch=1`, etc.
+
+## Download (pre-built binaries)
+
+Download the latest release for your platform from the [Releases](https://git.derickphan.com/lichenblankie/tinyweb/releases) page:
+
+| Platform | File |
+|----------|------|
+| Windows | `TinyWeb-windows-x64.exe` |
+| macOS | `TinyWeb-macos-arm64` |
+| Linux | `TinyWeb-linux-x64` |
+
+Run the downloaded file — no installation required.
+
+## Docker
+
+TinyWeb is distributed as source. Clone the repo, then build and run with Docker Compose:
+
+```bash
+git clone https://git.derickphan.com/lichenblankie/tinyweb.git
+cd tinyweb
+docker compose up -d
+```
+
+The bundled `docker-compose.yml` builds the image from source and persists your data in a named volume:
+
+```yaml
+services:
+  tinyweb:
+    build: .
+    ports:
+      - "8080:8080"
+    volumes:
+      - tinyweb-data:/data
+    restart: unless-stopped
+
+volumes:
+  tinyweb-data:
+```
+
+After the first build, the image is cached locally and subsequent `docker compose up -d` calls are instant. To update to the latest source:
+
+```bash
+git pull && docker compose up -d --build
+```
+
+If you're on macOS or need to reach a Reticulum node over TCP, uncomment the `RNS_TCP_HOST` / `RNS_TCP_PORT` block in `docker-compose.yml` and point it at a host running Reticulum. On Linux with LAN auto-discovery, leave it as-is (or switch to `network_mode: host`).
+
+### Storage Estimates
+
+Average web page content is ~15KB per page:
+
+| Pages | Database | Embeddings* | Total |
+|-------|----------|------------|-------|
+| 10,000 | 150MB | 80MB | ~250MB |
+| 100,000 | 1.5GB | 800MB | ~2.5GB |
+| 500,000 | 7.5GB | 4GB | ~12GB |
+| 1,000,000 | 15GB | 8GB | ~25GB |
+
+*Embeddings require semantic search to be enabled. With compression enabled (Settings > Search > AI), embeddings use ~50% less storage.
+
+Enable optional compression in Settings > Search > AI to reduce embedding storage by ~50%.
+
+## Data storage
+
+### Local (Python/binary)
+
+Your data is stored in `~/.tinyweb/`:
+
+| File | Description |
+|------|-------------|
+| `index.db` | SQLite database with your indexed pages |
+| `tinyweb_identity` | Your Reticulum identity (keep safe!) |
+| `forum.db` | Forum plugin database (only if forum is enabled) |
+| `models/` | Downloaded AI models for semantic search |
+| `index.hnsw` | Semantic search index |
+
+This allows your data to persist between upgrades and stay separate from the application.
+
+### Backups
+
+Back up the whole `~/.tinyweb/` directory periodically. The two files that matter:
+
+- **`tinyweb_identity`** is your permanent mesh identity. If you lose it, your destination hash changes and every subscriber has to re-subscribe to the new one. Keep it somewhere you trust; the file is `0600` by default.
+- **`index.db`** is your full reading history — every page, note, tag, and synced remote page. Losing it loses everything you've curated.
+- **`forum.db`** (if the forum plugin is enabled) — all threads, posts, upvotes, and moderation settings. Losing it loses your forum data.
+
+`models/` and `index.hnsw` are re-derivable (the model will re-download, and the HNSW index rebuilds from the database on next startup with semantic search enabled) so they don't need to be backed up.
+
+The `/export` page produces a JSON dump of your pages. It's a migration aid — it doesn't preserve your identity file, your custom template, or subscription state. A full restore needs a copy of `~/.tinyweb/`.
+
+### Docker
+
+When you run via `docker compose up` (above), data is stored in the `tinyweb-data` named volume and persists across rebuilds. To inspect or back up:
+
+```bash
+docker compose exec tinyweb ls -la /data
+docker compose down   # stop without removing the volume
+```
+
+To reset everything (destroys your index and identity — back up first):
+
+```bash
+docker compose down -v
+```
+
+### Command line options
+
+```bash
+./TinyWeb --version          # Show version
+./TinyWeb -p 9000            # Use port 9000 instead of default 8080
+./TinyWeb --bind 0.0.0.0     # Expose the web UI to your LAN (see warning below)
+```
+
+By default, the web UI binds to `127.0.0.1` and is only reachable from the machine running TinyWeb. **The UI has no authentication** — anyone who can reach the port can read, add, and delete entries, and change settings. Only pass `--bind 0.0.0.0` if you fully trust your network, or put TinyWeb behind an authenticating reverse proxy.
+
+## Getting started
+
+```bash
+pip install -r requirements.txt
+python app.py
+```
+
+This starts the Reticulum server and an HTTP gateway on `http://127.0.0.1:8080`. Open it in your browser. The UI is localhost-only by default; see `--bind` under *Command line options* if you want to reach it from another machine.
+
+Your destination hash is printed on startup — share it with friends so they can subscribe to your index.
+
+## Remote gateway
+
+To browse a remote TinyWeb instance without running your own index:
+
+```bash
+python gateway.py <destination_hash>
+```
+
+This connects over Reticulum and serves the remote instance at `http://localhost:8080`.
+
+## How it works
+
+1. **Save pages** — Use the `/add` form or the bookmarklet (found on `/style`) to index any URL
+2. **Search** — Full-text search across your saved pages, linked pages from trusted sites, and synced subscriptions
+3. **Subscribe** — Add a friend's destination hash on `/subscriptions` to sync their shared index
+4. **Customize** — Edit your site name, HTML template, and sharing settings on `/style`
+
+## Forum plugin
+
+TinyWeb ships with an optional [tinyweb-forum](https://git.derickphan.com/lichenblankie/tinyweb-forum) plugin — a decentralized link-sharing discussion board that runs in-process alongside TinyWeb.
+
+### Install
+
+```bash
+pip install tinyweb-forum
+```
+
+Enable it on the `/style` page under "Forum". A "Forum" link will appear in the navigation bar.
+
+### How it works
+
+- Threads and posts are stored in `~/.tinyweb/forum.db` (separate from your search index)
+- Instances are discovered automatically via mesh announces — no manual setup needed
+- Sync is manual by default: click "sync now" on the forum page. Auto-sync every 5 minutes is optional (toggle on moderation page)
+- At scale, sync uses epidemic gossip: 20 random peers per cycle, converging globally within ~O(log N) cycles
+- Authors are identified by a short pseudonymous identity hash (no accounts, no sign-up)
+- Auto-discovery can be disabled in the moderation page
+- Threads are auto-pruned after 30 days (configurable, or set to 0 to keep everything)
+- Moderation is local: block authors, mute threads, keyword filters, and gossip block lists with peers (auto-block after 3 peer reports)
+
+For full feature docs, see the [tinyweb-forum README](https://git.derickphan.com/lichenblankie/tinyweb-forum).
+
+## Project structure
+
+```
+app.py          — Entry point: boots Reticulum, starts HTTP gateway
+gateway.py      — HTTP-to-RNS bridge (local or remote dispatch)
+handlers.py     — Route dispatcher and all request handlers
+db.py           — SQLite database, FTS5, URL fetching, SSRF protection
+templates.py    — HTML template rendering and escaping
+rns_client.py   — Reticulum client for fetching remote site lists
+themes/         — Saved HTML templates (e.g. kodama.html)
+```
+
+## Security
+
+**The web UI has no authentication.** It is bound to `127.0.0.1` by default, so only processes on the local machine can reach it. If you pass `--bind 0.0.0.0` (or run inside a container with a published port), anyone who can reach that address can fully control your instance — reading private entries, changing settings, and modifying the HTML template (which runs in your browser). Put TinyWeb behind a reverse proxy with auth before exposing it beyond localhost.
+
+Other hardening measures:
+
+- **CSRF protection** — All POST forms use per-session tokens via double-submit cookies
+- **SSRF prevention** — URL fetching validates hostnames against private IP ranges, with redirect re-validation
+- **FTS5 injection prevention** — Search queries are sanitized before passing to SQLite MATCH
+- **Content Security Policy** — CSP headers on all HTML responses restrict script/style/frame sources
+- **XSS escaping** — All user-supplied content is HTML-escaped before rendering
+- **Bookmark authentication** — The bookmarklet endpoint requires a secret token
+- **Identity file protection** — The Reticulum identity key is restricted to owner-only permissions (0600)
+- **Forum caveats** — See [tinyweb-forum Security](https://git.derickphan.com/lichenblankie/tinyweb-forum#security) for forum-specific risks (voluntary retractions, block gossip manipulation, no rate limiting)
+
+## Maintenance
+
+### Database Vacuum
+
+Over time, deleted pages leave empty space in the database. Run the vacuum tool periodically to reclaim space:
+
+1. Go to `/style` in your browser
+2. Click "vacuum database" at the bottom of the page
+
+### Optional Compression
+
+To reduce storage for semantic search embeddings (~50% savings):
+
+1. Go to `/style` > Search > AI
+2. Enable "compress embeddings"
+3. Re-index your existing pages for the compression to apply to existing embeddings
+
+## Dependencies
+
+- [requests](https://docs.python-requests.org/) — HTTP fetching
+- [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/) — HTML parsing and link extraction
+- [rns](https://reticulum.network/) — Reticulum mesh networking
+
+## Philosophy
+
+TinyWeb is built for the slow web — intentionality over speed, human curation over algorithmic feeds, privacy over surveillance, and community over corporations. Every page in your index was saved because you found it valuable, not because an algorithm told you to click.

app.py

@@ -1,524 +1,314 @@
-import json
-import sqlite3
-import html
-import requests
-from http.server import HTTPServer, BaseHTTPRequestHandler
-from urllib.parse import parse_qs, urlparse, urljoin
-from bs4 import BeautifulSoup
+import os
+import sys
+import time
+import threading
+import argparse
+import RNS
+from http.server import HTTPServer
 
-DATABASE = "index.db"
+from db import init_db, get_setting, set_setting
+from handlers import dispatch_request
+import handlers as handlers_mod
+import templates as templates_mod
+import gateway
+from gateway import GatewayState, GatewayHandler
+
+APP_NAME = "tinyweb"
+ASPECTS = ["server"]
+IDENTITY_FILE = "tinyweb_identity"
+DEFAULT_TRANSPORT_HOST = "reticulum.derickphan.com"
+DEFAULT_TRANSPORT_PORT = 4242
+DATA_DIR = os.path.expanduser("~/.tinyweb")
 
 
-def get_db():
-    db = sqlite3.connect(DATABASE)
-    db.row_factory = sqlite3.Row
-    return db
+def get_transport_config():
+    host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST)
+    port = get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))
+    return host, int(port)
 
 
-def init_db():
-    db = sqlite3.connect(DATABASE)
-    db.execute(
-        "CREATE TABLE IF NOT EXISTS pages ("
-        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
-        "  url TEXT UNIQUE NOT NULL,"
-        "  title TEXT,"
-        "  body TEXT,"
-        "  note TEXT DEFAULT ''"
-        ")"
-    )
-    db.execute(
-        "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts "
-        "USING fts5(title, body, url, note, content=pages, content_rowid=id)"
-    )
-    db.execute(
-        "CREATE TABLE IF NOT EXISTS links ("
-        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
-        "  page_id INTEGER NOT NULL,"
-        "  url TEXT NOT NULL,"
-        "  label TEXT,"
-        "  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE"
-        ")"
-    )
-    db.execute(
-        "CREATE TABLE IF NOT EXISTS settings ("
-        "  key TEXT PRIMARY KEY,"
-        "  value TEXT"
-        ")"
-    )
-    db.executescript("""
-        CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
-            INSERT INTO pages_fts(rowid, title, body, url, note)
-            VALUES (new.id, new.title, new.body, new.url, new.note);
-        END;
-        CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
-            INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note)
-            VALUES ('delete', old.id, old.title, old.body, old.url, old.note);
-        END;
-        CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
-            INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note)
-            VALUES ('delete', old.id, old.title, old.body, old.url, old.note);
-            INSERT INTO pages_fts(rowid, title, body, url, note)
-            VALUES (new.id, new.title, new.body, new.url, new.note);
-        END;
-    """)
-    db.commit()
-    db.close()
-
-
-SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json")
-
-
-def fetch_page(url):
-    resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False)
-    resp.raise_for_status()
-    soup = BeautifulSoup(resp.text, "html.parser")
-
-    # extract links before stripping tags
-    domain = urlparse(url).netloc
-    seen = set()
-    links = []
-    for a in soup.find_all("a", href=True):
-        href = urljoin(url, a["href"]).split("#")[0]
-        parsed = urlparse(href)
-        if parsed.netloc != domain:
+def find_available_port(start=8080, max_attempts=20, host="127.0.0.1"):
+    """Find an available port starting from start."""
+    import socket
+    for port in range(start, start + max_attempts):
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                s.bind((host, port))
+            return port
+        except OSError:
             continue
-        if any(href.lower().endswith(ext) for ext in SKIP_EXT):
-            continue
-        if parsed.query or "action=" in href:
-            continue
-        path = parsed.path.lower()
-        if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")):
-            continue
-        if href in seen or href == url:
-            continue
-        seen.add(href)
-        label = a.get_text(strip=True) or href
-        links.append((href, label[:200]))
-
-    for tag in soup(["script", "style", "nav", "footer", "header"]):
-        tag.decompose()
-    title = soup.title.string.strip() if soup.title and soup.title.string else url
-    body = soup.get_text(separator=" ", strip=True)
-    return title, body, links
+    return start
 
 
-def snippet(text, query, ctx=80):
-    pos = text.lower().find(query.lower())
-    if pos == -1:
-        return text[:200]
-    start = max(0, pos - ctx)
-    end = min(len(text), pos + len(query) + ctx)
-    return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "")
+def get_version():
+    """Get version from git tag or VERSION file."""
+    try:
+        import subprocess
+        tag = subprocess.check_output(
+            ["git", "describe", "--tags", "--abbrev=0"],
+            stderr=subprocess.DEVNULL,
+            text=True
+        ).strip()
+        if tag.startswith("v"):
+            return tag[1:]
+        return tag
+    except Exception:
+        version_file = os.path.join(os.path.dirname(__file__), "VERSION")
+        if os.path.exists(version_file):
+            with open(version_file) as f:
+                return f.read().strip()
+        return "0.0.0"
 
 
-def esc(s):
-    return html.escape(str(s))
+def load_or_create_identity():
+    os.makedirs(DATA_DIR, exist_ok=True)
+    identity_path = os.path.join(DATA_DIR, IDENTITY_FILE)
+    if os.path.isfile(identity_path):
+        current = os.stat(identity_path).st_mode & 0o777
+        if current != 0o600:
+            os.chmod(identity_path, 0o600)
+        return RNS.Identity.from_file(identity_path)
+    identity = RNS.Identity()
+    identity.to_file(identity_path)
+    os.chmod(identity_path, 0o600)
+    return identity
 
 
-def get_setting(key, default=""):
-    db = get_db()
-    row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
-    db.close()
-    return row["value"] if row else default
+# Remote peers on the Reticulum mesh can only reach a narrow, read-only surface.
+# Any other method/path is rejected here — CSRF cannot authenticate mesh callers
+# (the attacker controls both the "cookie" and the "form" side of the check), so
+# gating by whitelist is the only safe option.
+_RNS_ALLOWED = {("GET", "/api/sites")}
 
 
-def set_setting(key, value):
-    db = get_db()
-    db.execute(
-        "INSERT INTO settings (key, value) VALUES (?, ?) "
-        "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
-        (key, value),
-    )
-    db.commit()
-    db.close()
+def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at):
+    if data is None:
+        data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""}
+    method = data.get("method", "GET")
+    req_path = data.get("path", "/")
+    if (method, req_path) not in _RNS_ALLOWED:
+        return {
+            "status": 403,
+            "content_type": "text/plain; charset=utf-8",
+            "body": "Forbidden: this endpoint is not available over Reticulum.",
+            "headers": {},
+        }
+    return dispatch_request(data)
 
 
-def get_site_name():
-    return get_setting("site_name", "tinyweb")
+def start_gateway(reticulum, bind_host="127.0.0.1"):
+    GatewayState.reticulum = reticulum
+    GatewayState.local_dispatch = dispatch_request
+    HTTPServer.allow_reuse_address = True
+    server = HTTPServer((bind_host, gateway.GATEWAY_PORT), GatewayHandler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
 
 
-def wrap_page(body_html):
-    css = get_setting("custom_css")
-    style = f"<style>{css}</style>" if css else ""
-    return f"<html><head>{style}</head><body>{body_html}</body></html>"
+def _config_settings_match(config_file, desired_host, desired_port):
+    """Check if existing config transport and LoRa settings match desired values."""
+    import configparser
+    try:
+        config = configparser.ConfigParser()
+        config.read(config_file)
+        # Check TCP transport
+        tcp_enabled = get_setting("tcp_enabled", "1") == "1"
+        has_tcp = config.has_section("TCP Transport")
+        if tcp_enabled != has_tcp:
+            return False
+        if tcp_enabled and has_tcp:
+            if (config.get("TCP Transport", "target_host") != desired_host or
+                    config.get("TCP Transport", "target_port") != str(desired_port)):
+                return False
+        # Check LoRa
+        lora_enabled = get_setting("lora_enabled", "0") == "1"
+        has_lora = config.has_section("RNode LoRa")
+        if lora_enabled != has_lora:
+            return False
+        if lora_enabled and has_lora:
+            if config.get("RNode LoRa", "port", fallback="") != get_setting("lora_port", ""):
+                return False
+            if config.get("RNode LoRa", "frequency", fallback="") != get_setting("lora_frequency", "867200000"):
+                return False
+        return True
+    except Exception:
+        pass
+    return False
 
 
-class Handler(BaseHTTPRequestHandler):
+def ensure_rns_config(config_dir, transport_host=None, transport_port=None):
+    """Generate a default Reticulum config with internet transport if none exists."""
+    if config_dir is None:
+        config_dir = os.path.expanduser("~/.reticulum")
+    config_file = os.path.join(config_dir, "config")
+    if transport_host is None:
+        transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST)
+    if transport_port is None:
+        transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT)))
 
-    def respond(self, body, status=200):
-        self.send_response(status)
-        self.send_header("Content-Type", "text/html; charset=utf-8")
-        self.end_headers()
-        self.wfile.write(wrap_page(body).encode())
-
-    def do_GET(self):
-        parsed = urlparse(self.path)
-        path = parsed.path
-        params = parse_qs(parsed.query)
-
-        if path == "/":
-            self.handle_search(params)
-        elif path == "/add":
-            self.handle_add_form()
-        elif path == "/pages":
-            self.handle_pages()
-        elif path.startswith("/delete/"):
-            self.handle_delete(path)
-        elif path.startswith("/edit/"):
-            self.handle_edit_form(path)
-        elif path == "/style":
-            self.handle_style_form()
-        elif path == "/bookmark":
-            self.handle_bookmark(params)
-        elif path == "/export":
-            self.handle_export()
-        elif path == "/import":
-            self.handle_import_form()
-        else:
-            self.respond("<h1>404</h1>", 404)
-
-    def do_POST(self):
-        length = int(self.headers.get("Content-Length", 0))
-        body = self.rfile.read(length).decode()
-        params = parse_qs(body)
-
-        if self.path == "/add":
-            self.handle_add_submit(params)
-        elif self.path.startswith("/edit/"):
-            self.handle_edit_submit(self.path, params)
-        elif self.path == "/style":
-            self.handle_style_submit(params)
-        elif self.path == "/import":
-            self.handle_import_submit(params)
-        else:
-            self.respond("<h1>404</h1>", 404)
-
-    def handle_search(self, params):
-        q = params.get("q", [""])[0].strip()
-        db = get_db()
-        count = db.execute("SELECT count(*) FROM pages").fetchone()[0]
-        name = get_site_name()
-
-        result_html = ""
-        trusted_html = ""
-        if q:
-            rows = db.execute(
-                "SELECT p.id, p.url, p.title, p.body, p.note "
-                "FROM pages_fts f JOIN pages p ON f.rowid = p.id "
-                "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50",
-                (q,),
-            ).fetchall()
-            if rows:
-                for r in rows:
-                    note_html = ""
-                    if r["note"]:
-                        note_html = f'<div class="note"><em>{esc(r["note"])}</em></div>'
-                    result_html += (
-                        f'<div class="result">'
-                        f'<a href="{esc(r["url"])}">{esc(r["title"])}</a><br>'
-                        f'<small>{esc(r["url"])}</small><br>'
-                        f'{esc(snippet(r["body"], q))}'
-                        f'{note_html}'
-                        f'</div>'
-                    )
-            else:
-                result_html = "<p>No results in your index.</p>"
-
-            # search all linked pages from trusted sites
-            words = q.lower().split()
-            all_links = db.execute(
-                "SELECT l.url, l.label, p.title AS source_title "
-                "FROM links l JOIN pages p ON l.page_id = p.id",
-            ).fetchall()
-            indexed_urls = set(r["url"] for r in rows) if rows else set()
-            seen = set()
-            trusted = []
-            for l in all_links:
-                if l["url"] in indexed_urls or l["url"] in seen:
-                    continue
-                if any(w in l["label"].lower() for w in words):
-                    seen.add(l["url"])
-                    trusted.append(l)
-                if len(trusted) >= 20:
-                    break
-
-            if trusted:
-                items = ""
-                for l in trusted:
-                    items += (
-                        f'<li><a href="{esc(l["url"])}">{esc(l["label"])}</a> '
-                        f'<small>— from {esc(l["source_title"])}</small></li>'
-                    )
-                trusted_html = (
-                    f'<details class="trusted">'
-                    f'<summary>from your trusted sites ({len(trusted)})</summary>'
-                    f'<ul>{items}</ul>'
-                    f'</details>'
+    managed_sentinel = "# managed by tinyweb"
+    if os.path.exists(config_file):
+        try:
+            with open(config_file) as f:
+                existing = f.read()
+        except OSError:
+            existing = ""
+        if managed_sentinel not in existing:
+            # User-authored config — don't clobber it.
+            if not _config_settings_match(config_file, transport_host, transport_port):
+                print(
+                    f"Warning: {config_file} was not created by tinyweb; "
+                    "leaving it alone. Edit it manually to change transport/LoRa settings."
                 )
-
-        db.close()
-        self.respond(
-            f'<h1><a href="/">{esc(name)}</a></h1>'
-            f'<form method="get" action="/">'
-            f'<input name="q" value="{esc(q)}" placeholder="search your index" size="40">'
-            f' <button type="submit">search</button>'
-            f'</form>'
-            f'<p>{count} page(s) indexed.'
-            f' <a href="/add">+ add url</a>'
-            f' | <a href="/pages">browse</a>'
-            f' | <a href="/style">customize</a></p>'
-            f'<hr>{result_html}{trusted_html}'
-        )
-
-    def handle_add_form(self, msg=""):
-        self.respond(
-            f"<h1>add url</h1>"
-            f'<form method="post" action="/add">'
-            f'<input name="url" placeholder="https://example.com" size="50"><br><br>'
-            f'<input name="note" placeholder="why are you saving this? (optional)" size="50"><br><br>'
-            f'<button type="submit">index</button>'
-            f"</form>"
-            f"<p>{msg}</p>"
-            f'<a href="/">back</a>'
-        )
-
-    def handle_add_submit(self, params):
-        url = params.get("url", [""])[0].strip()
-        note = params.get("note", [""])[0].strip()
-        if not url:
-            return self.handle_add_form("URL is required.")
-        if not url.startswith(("http://", "https://")):
-            return self.handle_add_form("URL must start with http:// or https://")
-        try:
-            title, body, links = fetch_page(url)
-            db = get_db()
-            cur = db.execute(
-                "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) "
-                "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note",
-                (url, title, body, note),
-            )
-            page_id = cur.lastrowid
-            db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
-            for href, label in links:
-                db.execute(
-                    "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)",
-                    (page_id, href, label),
-                )
-            db.commit()
-            db.close()
-            self.handle_add_form(f'Indexed: <a href="{esc(url)}">{esc(title)}</a>')
-        except Exception as e:
-            self.handle_add_form(f"Error: {esc(str(e))}")
-
-    def handle_pages(self):
-        db = get_db()
-        rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall()
-        db.close()
-        items = ""
-        for r in rows:
-            note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
-            items += (
-                f'<li>{esc(r["title"])}{note_html} '
-                f'<small>(<a href="{esc(r["url"])}">{esc(r["url"])}</a>)</small> '
-                f'<a href="/edit/{r["id"]}">edit</a> '
-                f'<a href="/delete/{r["id"]}">remove</a></li>'
-            )
-        self.respond(
-            f"<h1>indexed pages ({len(rows)})</h1>"
-            f"<ul>{items}</ul>"
-            f'<p><a href="/export">export</a> | <a href="/import">import</a></p>'
-            f'<a href="/">back</a>'
-        )
-
-    def handle_edit_form(self, path, msg=""):
-        try:
-            page_id = int(path.split("/")[-1])
-        except ValueError:
-            return self.respond("<h1>400</h1>", 400)
-        db = get_db()
-        row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone()
-        db.close()
-        if not row:
-            return self.respond("<h1>404</h1>", 404)
-        self.respond(
-            f"<h1>edit note</h1>"
-            f"<p><b>{esc(row['title'])}</b><br>"
-            f"<small>{esc(row['url'])}</small></p>"
-            f'<form method="post" action="/edit/{row["id"]}">'
-            f'<input name="note" value="{esc(row["note"])}" placeholder="why did you save this?" size="50"><br><br>'
-            f'<button type="submit">save</button>'
-            f"</form>"
-            f"<p>{msg}</p>"
-            f'<a href="/pages">back</a>'
-        )
-
-    def handle_edit_submit(self, path, params):
-        try:
-            page_id = int(path.split("/")[-1])
-        except ValueError:
-            return self.respond("<h1>400</h1>", 400)
-        note = params.get("note", [""])[0].strip()
-        db = get_db()
-        db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id))
-        db.commit()
-        db.close()
-        self.send_response(302)
-        self.send_header("Location", "/pages")
-        self.end_headers()
-
-    def handle_delete(self, path):
-        try:
-            page_id = int(path.split("/")[-1])
-        except ValueError:
-            return self.respond("<h1>400</h1>", 400)
-        db = get_db()
-        db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
-        db.execute("DELETE FROM pages WHERE id = ?", (page_id,))
-        db.commit()
-        db.close()
-        self.send_response(302)
-        self.send_header("Location", "/pages")
-        self.end_headers()
-
-    def handle_bookmark(self, params):
-        url = params.get("url", [""])[0].strip()
-        if not url or not url.startswith(("http://", "https://")):
-            self.send_response(200)
-            self.send_header("Content-Type", "text/plain")
-            self.send_header("Access-Control-Allow-Origin", "*")
-            self.end_headers()
-            self.wfile.write(b"error: invalid url")
             return
-        try:
-            title, body, links = fetch_page(url)
-            db = get_db()
-            cur = db.execute(
-                "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, '') "
-                "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body",
-                (url, title, body),
-            )
-            page_id = cur.lastrowid
-            db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
-            for href, label in links:
-                db.execute(
-                    "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)",
-                    (page_id, href, label),
-                )
-            db.commit()
-            db.close()
-            msg = f"ok: {title}"
-        except Exception as e:
-            msg = f"error: {e}"
-        self.send_response(200)
-        self.send_header("Content-Type", "text/plain")
-        self.send_header("Access-Control-Allow-Origin", "*")
-        self.end_headers()
-        self.wfile.write(msg.encode())
+        if _config_settings_match(config_file, transport_host, transport_port):
+            return
 
-    def handle_export(self):
-        db = get_db()
-        rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall()
-        db.close()
-        data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows]
-        self.send_response(200)
-        self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Disposition", "attachment; filename=tinyweb-export.json")
-        self.end_headers()
-        self.wfile.write(json.dumps(data, indent=2).encode())
+    # Build optional interface blocks
+    tcp_block = ""
+    if get_setting("tcp_enabled", "1") == "1":
+        tcp_block = f"""
+  [[TCP Transport]]
+    type = TCPClientInterface
+    enabled = yes
+    target_host = {transport_host}
+    target_port = {transport_port}
+"""
 
-    def handle_import_form(self, msg=""):
-        self.respond(
-            f"<h1>import</h1>"
-            f"<p>Paste the contents of a tinyweb export file (JSON).</p>"
-            f'<form method="post" action="/import">'
-            f'<textarea name="data" rows="12" cols="60" placeholder=\'[{{"url": "...", "note": "..."}}]\'></textarea><br><br>'
-            f'<button type="submit">import</button>'
-            f"</form>"
-            f"<p>{msg}</p>"
-            f'<a href="/pages">back</a>'
-        )
+    lora_block = ""
+    if get_setting("lora_enabled", "0") == "1":
+        lora_port = get_setting("lora_port", "")
+        if lora_port:
+            lora_frequency = get_setting("lora_frequency", "867200000")
+            lora_bandwidth = get_setting("lora_bandwidth", "125000")
+            lora_txpower = get_setting("lora_txpower", "7")
+            lora_sf = get_setting("lora_sf", "8")
+            lora_cr = get_setting("lora_cr", "5")
+            lora_block = f"""
+  [[RNode LoRa]]
+    type = RNodeInterface
+    enabled = yes
+    port = {lora_port}
+    frequency = {lora_frequency}
+    bandwidth = {lora_bandwidth}
+    txpower = {lora_txpower}
+    spreadingfactor = {lora_sf}
+    codingrate = {lora_cr}
+"""
 
-    def handle_import_submit(self, params):
-        raw = params.get("data", [""])[0].strip()
-        if not raw:
-            return self.handle_import_form("Paste JSON data.")
-        try:
-            data = json.loads(raw)
-        except json.JSONDecodeError:
-            return self.handle_import_form("Invalid JSON.")
-        if not isinstance(data, list):
-            return self.handle_import_form("Expected a JSON array.")
+    os.makedirs(config_dir, exist_ok=True)
+    with open(config_file, "w") as f:
+        f.write(f"""{managed_sentinel}
+[reticulum]
+  enable_transport = False
+  share_instance = No
 
-        imported = 0
-        errors = 0
-        for entry in data:
-            url = entry.get("url", "").strip()
-            note = entry.get("note", "").strip()
-            if not url:
-                continue
-            try:
-                title, body, links = fetch_page(url)
-                db = get_db()
-                cur = db.execute(
-                    "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) "
-                    "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note",
-                    (url, title, body, note),
-                )
-                page_id = cur.lastrowid
-                db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
-                for href, label in links:
-                    db.execute(
-                        "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)",
-                        (page_id, href, label),
-                    )
-                db.commit()
-                db.close()
-                imported += 1
-            except Exception:
-                errors += 1
+[logging]
+  loglevel = 4
 
-        self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).")
+[interfaces]
+  [[Default Interface]]
+    type = AutoInterface
+    enabled = Yes
+{tcp_block}{lora_block}""")
+    print(f"Created Reticulum config at {config_file}")
 
-    def handle_style_form(self, msg=""):
-        css = get_setting("custom_css")
-        name = get_site_name()
-        self.respond(
-            f"<h1>customize</h1>"
-            f"<h2>name your search engine</h2>"
-            f'<form method="post" action="/style">'
-            f'<input name="site_name" value="{esc(name)}" placeholder="tinyweb" size="30"><br><br>'
-            f"<h2>custom css</h2>"
-            f"<p>Some classes you can target:</p>"
-            f"<pre>"
-            f"body          - page background, font\n"
-            f"h1            - page titles\n"
-            f"input, button - search bar\n"
-            f"a             - links\n"
-            f".result       - each search result\n"
-            f".note         - your notes on results\n"
-            f".trusted      - trusted sites dropdown\n"
-            f"small         - url text\n"
-            f"ul, li        - browse page list"
-            f"</pre>"
-            f'<textarea name="css" rows="16" cols="60">{esc(css)}</textarea><br><br>'
-            f'<button type="submit">save</button>'
-            f"</form>"
-            f"<h2>bookmarklet</h2>"
-            f"<p>Drag this link to your bookmarks bar. Click it on any page to index it instantly.</p>"
-            f'<p><a href="javascript:void(fetch(\'http://localhost:5001/bookmark?url=\'+encodeURIComponent(location.href)).then(r=>r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}</a></p>'
-            f"<p>{msg}</p>"
-            f'<a href="/">back</a>'
-        )
 
-    def handle_style_submit(self, params):
-        css = params.get("css", [""])[0]
-        name = params.get("site_name", ["tinyweb"])[0].strip()
-        set_setting("custom_css", css)
-        set_setting("site_name", name or "tinyweb")
-        self.handle_style_form("Saved.")
+def _preload_embeddings():
+    """Pre-load the embedding model and build the HNSW index in background."""
+    if get_setting("semantic_search", "0") != "1":
+        print("Semantic search disabled.")
+        return
+    try:
+        from embeddings import _get_session, _get_reranker, build_index
+        _get_session()
+        build_index()
+        if get_setting("use_reranker", "0") == "1":
+            _get_reranker()
+            print("Semantic search ready (with reranker).")
+        else:
+            print("Semantic search ready.")
+    except Exception as e:
+        print(f"Semantic search unavailable: {e}")
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="tinyweb", description="Personal decentralized search engine")
+    parser.add_argument("--version", "-v", action="store_true", help="Show version")
+    parser.add_argument("--port", "-p", type=int, default=None, help="HTTP gateway port (default: 8080)")
+    parser.add_argument(
+        "--bind", "-b", default="127.0.0.1",
+        help="Address to bind the HTTP gateway to (default: 127.0.0.1). "
+             "Use 0.0.0.0 to expose to the LAN; note that the web UI has no authentication.",
+    )
+    args = parser.parse_args()
+
+    if args.version:
+        print(f"TinyWeb {get_version()}")
+        return
+
+    bind_host = args.bind
+    port = args.port or 8080
+    gateway.GATEWAY_PORT = find_available_port(port, host=bind_host)
+
+    init_db()
+    transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST)
+    transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT)))
+    threading.Thread(target=_preload_embeddings, daemon=True).start()
+    config_dir = os.environ.get("RNS_CONFIG_DIR")
+    ensure_rns_config(config_dir, transport_host, transport_port)
+    reticulum = RNS.Reticulum(configdir=config_dir)
+    identity = load_or_create_identity()
+
+    destination = RNS.Destination(
+        identity,
+        RNS.Destination.IN,
+        RNS.Destination.SINGLE,
+        APP_NAME,
+        *ASPECTS,
+    )
+
+    destination.register_request_handler(
+        "/tinyweb",
+        response_generator=rns_request_handler,
+        allow=RNS.Destination.ALLOW_ALL,
+    )
+
+    # Initialize forum plugin if available
+    forum = None
+    try:
+        from tinyweb_forum import ForumPlugin
+        from db import get_site_name
+        forum = ForumPlugin(DATA_DIR, identity, reticulum, site_name=get_site_name())
+        if get_setting("forum_enabled", "0") == "1":
+            forum.enable()
+            templates_mod.FORUM_ENABLED = True
+        handlers_mod.forum_plugin = forum
+        print(f"Forum plugin: {'enabled' if forum.is_enabled() else 'available (enable in settings)'}")
+    except ImportError:
+        print("Forum plugin not installed (pip install tinyweb[forum])")
+    except Exception as e:
+        print(f"Forum plugin error: {e}")
+
+    # Brief delay to ensure all interfaces (especially TCP) are fully ready
+    time.sleep(2)
+    destination.announce()
+    set_setting("dest_hash", destination.hash.hex())
+    start_gateway(reticulum, bind_host=bind_host)
+
+    print(f"TinyWeb running!")
+    if bind_host in ("0.0.0.0", "::"):
+        print(f"Open http://localhost:{gateway.GATEWAY_PORT} in your browser")
+        print(f"WARNING: listening on {bind_host} — the web UI has no authentication. "
+              "Anyone on your network can control this instance.")
+    else:
+        print(f"Open http://{bind_host}:{gateway.GATEWAY_PORT} in your browser")
+    print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)")
+
+    while True:
+        time.sleep(1)
 
 
 if __name__ == "__main__":
-    init_db()
-    print("running on http://localhost:5001")
-    HTTPServer(("localhost", 5001), Handler).serve_forever()
+    main()

conftest.py

@@ -0,0 +1,128 @@
+"""Shared pytest fixtures for TinyWeb tests.
+
+Three fixtures cover most tests: `temp_db` swaps the SQLite path to a
+per-test tempfile, `seeded_db` layers sample rows on top, and `csrf_session`
+primes the thread-local CSRF token that handlers read.
+"""
+import socket
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+import db as db_module
+import handlers as handlers_module
+
+
+@pytest.fixture
+def temp_db(tmp_path, monkeypatch):
+    """Isolated SQLite DB per test.
+
+    Swaps `db.DATABASE` and `db.DATA_DIR` to a tempdir, clears the connection
+    pool before and after so state doesn't leak across tests, and calls
+    `init_db()` so every schema object exists.
+    """
+    data_dir = tmp_path / "tinyweb"
+    data_dir.mkdir()
+    db_path = data_dir / "index.db"
+
+    monkeypatch.setattr(db_module, "DATA_DIR", str(data_dir))
+    monkeypatch.setattr(db_module, "DATABASE", str(db_path))
+
+    with db_module._pool_lock:
+        for conn in db_module._pool:
+            try:
+                conn.close()
+            except Exception:
+                pass
+        db_module._pool.clear()
+
+    db_module.init_db()
+    yield db_path
+
+    with db_module._pool_lock:
+        for conn in db_module._pool:
+            try:
+                conn.close()
+            except Exception:
+                pass
+        db_module._pool.clear()
+
+
+@pytest.fixture
+def seeded_db(temp_db):
+    """A temp DB with a small, realistic set of pages/tags/links."""
+    db = db_module.get_db()
+    try:
+        rows = [
+            ("https://example.com/rust-intro", "Rust Intro", "A gentle introduction to rust borrow checker.", "notes on ownership"),
+            ("https://example.com/python-tips", "Python Tips", "Daily python tricks for readable code.", ""),
+            ("https://example.com/ocaml-why", "Why OCaml", "Type systems and inference in ocaml.", "private thoughts"),
+            ("https://news.example.org/mesh", "Mesh Networking", "Reticulum and LoRa for decentralized networks.", ""),
+        ]
+        for url, title, body, note in rows:
+            db.execute(
+                "INSERT INTO pages (url, title, body, note, last_modified) "
+                "VALUES (?, ?, ?, ?, '2026-04-01T00:00:00')",
+                (url, title, body, note),
+            )
+        db.commit()
+        page_ids = {
+            row["url"]: row["id"]
+            for row in db.execute("SELECT id, url FROM pages").fetchall()
+        }
+        tag_rows = [
+            (page_ids["https://example.com/rust-intro"], ["rust", "public"]),
+            (page_ids["https://example.com/python-tips"], ["python"]),
+            (page_ids["https://example.com/ocaml-why"], ["ocaml", "private"]),
+            (page_ids["https://news.example.org/mesh"], ["mesh", "public"]),
+        ]
+        for pid, tags in tag_rows:
+            for name in tags:
+                db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,))
+                tid = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()[0]
+                db.execute(
+                    "INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)",
+                    (pid, tid),
+                )
+        db.execute(
+            "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)",
+            (page_ids["https://example.com/rust-intro"], "https://example.com/rust-advanced", "advanced rust guide"),
+        )
+        db.commit()
+    finally:
+        db_module.return_db(db)
+    return temp_db
+
+
+@pytest.fixture
+def csrf_session(monkeypatch):
+    """Prime the CSRF thread-local so handler code that calls _get_csrf_token works."""
+    token = "test-csrf-token"
+    handlers_module._request_local.csrf_token = token
+    yield token
+    if hasattr(handlers_module._request_local, "csrf_token"):
+        del handlers_module._request_local.csrf_token
+
+
+def patch_dns_fail(monkeypatch):
+    """Make every socket.getaddrinfo call raise gaierror for the rest of this test."""
+    def boom(*args, **kwargs):
+        raise socket.gaierror("test: DNS disabled")
+    monkeypatch.setattr(socket, "getaddrinfo", boom)
+
+
+def patch_dns_ok(monkeypatch, address="93.184.216.34"):
+    """Make every getaddrinfo return a single public IP for the rest of this test."""
+    def ok(host, port, *args, **kwargs):
+        return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (address, port or 80))]
+    monkeypatch.setattr(socket, "getaddrinfo", ok)
+
+
+def patch_dns_private(monkeypatch, address="127.0.0.1"):
+    """Make every getaddrinfo return a private/blocked IP for the rest of this test."""
+    def private(host, port, *args, **kwargs):
+        return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (address, port or 80))]
+    monkeypatch.setattr(socket, "getaddrinfo", private)

db.py

@@ -0,0 +1,449 @@
+import socket
+import ipaddress
+import sqlite3
+import requests
+import os
+from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse, quote
+from bs4 import BeautifulSoup
+
+DATA_DIR = os.path.expanduser("~/.tinyweb")
+DATABASE = os.path.join(DATA_DIR, "index.db")
+
+BLOCKED_NETWORKS = [
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("0.0.0.0/8"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fc00::/7"),
+    ipaddress.ip_network("fe80::/10"),
+]
+
+
+def _is_blocked_response(html, status_code):
+    """Check if response is a CDN challenge/block page."""
+    if status_code == 403:
+        return True
+    html_lower = html.lower()
+    if "just a moment" in html_lower or "cloudflare" in html_lower:
+        return True
+    if "enable javascript and cookies" in html_lower:
+        return True
+    if "request rejected" in html_lower:
+        return True
+    if "access denied" in html_lower:
+        return True
+    return False
+
+
+def _validate_url_target(url):
+    """Resolve hostname and block private/internal IPs to prevent SSRF."""
+    parsed = urlparse(url)
+    hostname = parsed.hostname
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    if not hostname:
+        raise ValueError(f"No hostname in URL: {url}")
+    try:
+        addrs = socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP)
+    except socket.gaierror:
+        raise ValueError(f"Cannot resolve hostname: {hostname}")
+    for family, type_, proto, canonname, sockaddr in addrs:
+        ip = ipaddress.ip_address(sockaddr[0])
+        for network in BLOCKED_NETWORKS:
+            if ip in network:
+                raise ValueError(f"URL resolves to blocked address: {ip}")
+
+SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json")
+
+TRACKING_PARAMS = {
+    "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
+    "fbclid", "gclid", "msclkid", "mc_cid", "mc_eid", "ref", "ref_src",
+    "ref_url", "_ga", "_gl", "yclid", "twclid", "igshid",
+}
+
+
+def clean_url(url):
+    parsed = urlparse(url)
+
+    # Prefer https
+    scheme = "https" if parsed.scheme in ("http", "https") else parsed.scheme
+
+    # Normalize hostname: lowercase, strip www (only if non-www resolves)
+    hostname = (parsed.hostname or "").lower()
+    original_hostname = hostname
+    if hostname.startswith("www."):
+        hostname = hostname[4:]
+        port = parsed.port or (443 if scheme == "https" else 80)
+        try:
+            socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP)
+        except socket.gaierror:
+            hostname = original_hostname
+
+    # Preserve explicit non-default ports
+    port = parsed.port
+    if port and ((scheme == "https" and port == 443) or (scheme == "http" and port == 80)):
+        port = None
+    netloc = f"{hostname}:{port}" if port else hostname
+
+    # Strip trailing slash (keep root "/" as-is)
+    path = parsed.path.rstrip("/") or "/"
+
+    # Remove tracking params and sort remaining for consistent ordering
+    params = parse_qs(parsed.query)
+    cleaned = sorted(
+        ((k, sorted(v)) for k, v in params.items() if k.lower() not in TRACKING_PARAMS),
+        key=lambda x: x[0],
+    )
+    new_query = urlencode(cleaned, doseq=True, quote_via=quote)
+
+    return urlunparse((scheme, netloc, path, "", new_query, ""))
+
+
+_pool = []
+_pool_lock = __import__("threading").Lock()
+_POOL_SIZE = 16
+
+
+def get_db():
+    with _pool_lock:
+        if _pool:
+            db = _pool.pop()
+            try:
+                db.execute("SELECT 1")
+                return db
+            except Exception:
+                pass
+    db = sqlite3.connect(DATABASE, timeout=10)
+    db.execute("PRAGMA journal_mode=WAL")
+    db.execute("PRAGMA foreign_keys = ON")
+    db.row_factory = sqlite3.Row
+    return db
+
+
+def return_db(db):
+    try:
+        db.rollback()
+    except Exception:
+        try:
+            db.close()
+        except Exception:
+            pass
+        return
+    with _pool_lock:
+        if len(_pool) < _POOL_SIZE:
+            _pool.append(db)
+        else:
+            db.close()
+
+
+def init_db():
+    os.makedirs(DATA_DIR, exist_ok=True)
+    db = sqlite3.connect(DATABASE)
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS pages ("
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "  url TEXT UNIQUE NOT NULL,"
+        "  title TEXT,"
+        "  body TEXT,"
+        "  note TEXT DEFAULT '',"
+        "  last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now')),"
+        "  reticulum_dest TEXT DEFAULT ''"
+        ")"
+    )
+    db.execute(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts "
+        "USING fts5(title, body, url, note, content=pages, content_rowid=id)"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS links ("
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "  page_id INTEGER NOT NULL,"
+        "  url TEXT NOT NULL,"
+        "  label TEXT,"
+        "  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE"
+        ")"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS settings ("
+        "  key TEXT PRIMARY KEY,"
+        "  value TEXT"
+        ")"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS subscriptions ("
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "  dest_hash TEXT UNIQUE NOT NULL,"
+        "  name TEXT DEFAULT '',"
+        "  auto_sync INTEGER DEFAULT 0,"
+        "  last_sync TEXT DEFAULT ''"
+        ")"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS remote_pages ("
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "  subscription_id INTEGER NOT NULL,"
+        "  url TEXT NOT NULL,"
+        "  title TEXT,"
+        "  note TEXT DEFAULT '',"
+        "  tags TEXT DEFAULT '',"
+        "  FOREIGN KEY (subscription_id) REFERENCES subscriptions(id) ON DELETE CASCADE,"
+        "  UNIQUE(subscription_id, url)"
+        ")"
+    )
+    db.execute(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS remote_pages_fts "
+        "USING fts5(title, url, note, content=remote_pages, content_rowid=id)"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS tags ("
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "  name TEXT UNIQUE NOT NULL"
+        ")"
+    )
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS page_tags ("
+        "  page_id INTEGER NOT NULL,"
+        "  tag_id INTEGER NOT NULL,"
+        "  PRIMARY KEY (page_id, tag_id),"
+        "  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE,"
+        "  FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE"
+        ")"
+    )
+    db.executescript("""
+        CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
+            INSERT INTO pages_fts(rowid, title, body, url, note)
+            VALUES (new.id, new.title, new.body, new.url, new.note);
+        END;
+        CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
+            INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note)
+            VALUES ('delete', old.id, old.title, old.body, old.url, old.note);
+        END;
+        CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
+            INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note)
+            VALUES ('delete', old.id, old.title, old.body, old.url, old.note);
+            INSERT INTO pages_fts(rowid, title, body, url, note)
+            VALUES (new.id, new.title, new.body, new.url, new.note);
+        END;
+        CREATE TRIGGER IF NOT EXISTS remote_pages_ai AFTER INSERT ON remote_pages BEGIN
+            INSERT INTO remote_pages_fts(rowid, title, url, note)
+            VALUES (new.id, new.title, new.url, new.note);
+        END;
+        CREATE TRIGGER IF NOT EXISTS remote_pages_ad AFTER DELETE ON remote_pages BEGIN
+            INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note)
+            VALUES ('delete', old.id, old.title, old.url, old.note);
+        END;
+        CREATE TRIGGER IF NOT EXISTS remote_pages_au AFTER UPDATE ON remote_pages BEGIN
+            INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note)
+            VALUES ('delete', old.id, old.title, old.url, old.note);
+            INSERT INTO remote_pages_fts(rowid, title, url, note)
+            VALUES (new.id, new.title, new.url, new.note);
+        END;
+    """)
+    # Migrate old subscriptions table if needed
+    cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()]
+    if "url" in cols and "dest_hash" not in cols:
+        db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash")
+        db.commit()
+
+    # Migrate remote_pages: add tags column if missing
+    rp_cols = [row[1] for row in db.execute("PRAGMA table_info(remote_pages)").fetchall()]
+    if "tags" not in rp_cols:
+        db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''")
+        db.commit()
+
+    # Migrate pages: add last_modified column if missing
+    page_cols = [row[1] for row in db.execute("PRAGMA table_info(pages)").fetchall()]
+    if "last_modified" not in page_cols:
+        db.execute("ALTER TABLE pages ADD COLUMN last_modified TEXT DEFAULT ''")
+        db.execute("UPDATE pages SET last_modified = strftime('%Y-%m-%dT%H:%M:%S','now') WHERE last_modified = ''")
+        db.commit()
+
+    # Migrate pages: add summary column if missing
+    if "summary" not in page_cols:
+        db.execute("ALTER TABLE pages ADD COLUMN summary TEXT DEFAULT ''")
+        db.commit()
+
+    # Migrate pages: add reticulum_dest column if missing
+    if "reticulum_dest" not in page_cols:
+        db.execute("ALTER TABLE pages ADD COLUMN reticulum_dest TEXT DEFAULT ''")
+        db.commit()
+
+    # Chunks table for semantic search embeddings
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS chunks ("
+        "  id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        "  page_id INTEGER,"
+        "  remote_page_id INTEGER,"
+        "  chunk_index INTEGER NOT NULL,"
+        "  chunk_text TEXT NOT NULL,"
+        "  embedding BLOB NOT NULL,"
+        "  FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE,"
+        "  FOREIGN KEY (remote_page_id) REFERENCES remote_pages(id) ON DELETE CASCADE"
+        ")"
+    )
+    db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page ON chunks(page_id)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_remote ON chunks(remote_page_id)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page_idx ON chunks(page_id, chunk_index)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_pages_url ON pages(url)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_pages_modified ON pages(last_modified)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_page ON page_tags(page_id)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_tag ON page_tags(tag_id)")
+
+    # Migrate custom_template: replace hardcoded forum link with {{forum_link}} placeholder
+    cur = db.execute("SELECT value FROM settings WHERE key='custom_template'")
+    row = cur.fetchone()
+    if row:
+        updated = row[0].replace('<a href="/forum">forum</a>', "{{forum_link}}")
+        if updated != row[0]:
+            db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,))
+            db.commit()
+
+    # Migrate custom_template: replace hardcoded site name with {{site_name}} placeholder
+    cur = db.execute("SELECT value FROM settings WHERE key='custom_template'")
+    row = cur.fetchone()
+    if row and '{{site_name}}' not in row[0]:
+        updated = row[0].replace('href="/">tinyweb</a>', 'href="/">{{site_name}}</a>')
+        if updated != row[0]:
+            db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,))
+            db.commit()
+
+    db.execute("PRAGMA journal_mode=WAL")
+    db.execute("PRAGMA synchronous=NORMAL")
+    db.execute("PRAGMA cache_size=-64000")
+    db.commit()
+    db.close()
+
+
+def get_setting(key, default=""):
+    db = get_db()
+    try:
+        row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
+        return row["value"] if row else default
+    finally:
+        return_db(db)
+
+
+def vacuum_db():
+    """Run VACUUM and WAL checkpoint to reclaim space after deletions."""
+    db = get_db()
+    try:
+        db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        db.execute("VACUUM")
+    finally:
+        return_db(db)
+
+
+def set_setting(key, value):
+    db = get_db()
+    try:
+        db.execute(
+            "INSERT INTO settings (key, value) VALUES (?, ?) "
+            "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+            (key, value),
+        )
+        db.commit()
+    finally:
+        return_db(db)
+
+
+def get_site_name():
+    return get_setting("site_name", "tinyweb")
+
+
+def fetch_page(url):
+    _validate_url_target(url)
+    resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False)
+
+    if _is_blocked_response(resp.text, resp.status_code):
+        raise Exception(f"Site blocks automated access: {resp.status_code}")
+
+    # Follow redirects manually, re-validating each target
+    max_redirects = 5
+    while resp.is_redirect and max_redirects > 0:
+        redirect_url = resp.headers.get("Location")
+        if not redirect_url:
+            break
+        redirect_url = urljoin(url, redirect_url)
+        _validate_url_target(redirect_url)
+        url = redirect_url
+        resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False)
+        max_redirects -= 1
+    resp.raise_for_status()
+    soup = BeautifulSoup(resp.text, "html.parser")
+
+    # extract links before stripping tags
+    domain = urlparse(url).netloc
+    seen = set()
+    links = []
+    for a in soup.find_all("a", href=True):
+        href = urljoin(url, a["href"]).split("#")[0]
+        parsed = urlparse(href)
+        if parsed.netloc != domain:
+            continue
+        if any(href.lower().endswith(ext) for ext in SKIP_EXT):
+            continue
+        if parsed.query or "action=" in href:
+            continue
+        path = parsed.path.lower()
+        if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")):
+            continue
+        if href in seen or href == url:
+            continue
+        seen.add(href)
+        label = a.get_text(strip=True) or href
+        links.append((href, label[:200]))
+
+    # Extract meta description before stripping tags (case-insensitive)
+    meta_desc = ""
+    for m in soup.find_all("meta"):
+        name = (m.get("name") or "").lower()
+        prop = (m.get("property") or "").lower()
+        content = (m.get("content") or "").strip()
+        if not content:
+            continue
+        if name == "description" and len(content) > len(meta_desc):
+            meta_desc = content
+        elif prop == "og:description" and not meta_desc:
+            meta_desc = content
+
+    for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]):
+        tag.decompose()
+    title = soup.title.string.strip() if soup.title and soup.title.string else url
+
+    body = soup.get_text(separator=" ", strip=True)
+    return title, body, links, meta_desc
+
+
+
+def index_url(url, note="", reticulum_dest=""):
+    url = clean_url(url)
+    title, body, links, meta_desc = fetch_page(url)
+    summary = meta_desc if meta_desc and len(meta_desc) > 20 else ""
+    db = get_db()
+    try:
+        now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        db.execute(
+            "INSERT INTO pages (url, title, body, note, last_modified, summary, reticulum_dest) VALUES (?, ?, ?, ?, ?, ?, ?) "
+            "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, "
+            "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary, reticulum_dest=excluded.reticulum_dest",
+            (url, title, body, note, now, summary, reticulum_dest),
+        )
+        page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0]
+        db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
+        for href, label in links:
+            db.execute(
+                "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)",
+                (page_id, href, label),
+            )
+        db.commit()
+        if get_setting("semantic_search", "0") == "1":
+            try:
+                from embeddings import store_embeddings
+                store_embeddings(page_id, title, body, db)
+            except Exception:
+                pass  # embedding generation is best-effort
+    finally:
+        return_db(db)
+    return title

docker-compose.yml

@@ -0,0 +1,17 @@
+services:
+  tinyweb:
+    build: .
+    ports:
+      - "8080:8080"
+    volumes:
+      - tinyweb-data:/data
+    restart: unless-stopped
+    # Connect to another Reticulum instance over TCP.
+    # Required on macOS (Docker can't do LAN auto-discovery).
+    # On Linux, auto-discovery works with network_mode: host.
+    # environment:
+    #   - RNS_TCP_HOST=10.0.0.100
+    #   - RNS_TCP_PORT=4242
+
+volumes:
+  tinyweb-data:

embeddings.py

@@ -0,0 +1,586 @@
+"""Semantic search using Snowflake arctic-embed-s via ONNX Runtime + hnswlib."""
+
+import os
+import re
+import threading
+import numpy as np
+
+DATA_DIR = os.path.expanduser("~/.tinyweb")
+MODEL_ID = "Snowflake/snowflake-arctic-embed-s"
+MODEL_DIR = os.path.join(DATA_DIR, "models", "snowflake-arctic-embed-s")
+RERANKER_DIR = os.path.join(DATA_DIR, "models", "cross-encoder")
+HNSW_PATH = os.path.join(DATA_DIR, "index.hnsw")
+DIMS = 384
+MAX_TOKENS = 512
+QUERY_PREFIX = "Represent this sentence for searching relevant passages: "
+
+_session = None
+_tokenizer = None
+_lock = threading.Lock()
+
+_reranker_session = None
+_reranker_tokenizer = None
+_reranker_lock = threading.Lock()
+
+# Live HNSW index and chunk-id mapping
+_hnsw_index = None
+_hnsw_ids = []  # maps internal HNSW label -> chunks.id
+_hnsw_lock = threading.Lock()
+
+
+# ---------------------------------------------------------------------------
+# Model download & loading
+# ---------------------------------------------------------------------------
+
+def _ensure_model():
+    """Download the ONNX model and tokenizer from HuggingFace if not present."""
+    os.makedirs(MODEL_DIR, exist_ok=True)
+    model_path = os.path.join(MODEL_DIR, "model.onnx")
+    tokenizer_path = os.path.join(MODEL_DIR, "tokenizer.json")
+    if os.path.exists(model_path) and os.path.exists(tokenizer_path):
+        return
+    from huggingface_hub import hf_hub_download
+    os.makedirs(MODEL_DIR, exist_ok=True)
+    files = {
+        "onnx/model_quantized.onnx": "model.onnx",
+        "tokenizer.json": "tokenizer.json",
+        "tokenizer_config.json": "tokenizer_config.json",
+    }
+    for remote, local in files.items():
+        target = os.path.join(MODEL_DIR, local)
+        if os.path.exists(target):
+            continue
+        cached = hf_hub_download(repo_id=MODEL_ID, filename=remote)
+        # hf_hub_download returns the cached file path; copy to our model dir
+        import shutil
+        shutil.copy2(cached, target)
+
+
+def _get_session():
+    """Return (onnxruntime.InferenceSession, tokenizers.Tokenizer) singleton."""
+    global _session, _tokenizer
+    if _session is not None:
+        return _session, _tokenizer
+    with _lock:
+        if _session is not None:
+            return _session, _tokenizer
+        _ensure_model()
+        import onnxruntime as ort
+        from tokenizers import Tokenizer
+        _session = ort.InferenceSession(
+            os.path.join(MODEL_DIR, "model.onnx"),
+            providers=["CPUExecutionProvider"],
+        )
+        _tokenizer = Tokenizer.from_file(os.path.join(MODEL_DIR, "tokenizer.json"))
+        _tokenizer.enable_truncation(max_length=MAX_TOKENS)
+        _tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=None)
+        return _session, _tokenizer
+
+
+def _get_reranker():
+    """Return (onnxruntime.InferenceSession, tokenizers.Tokenizer) for the cross-encoder reranker."""
+    global _reranker_session, _reranker_tokenizer
+    if _reranker_session is not None:
+        return _reranker_session, _reranker_tokenizer
+    with _reranker_lock:
+        if _reranker_session is not None:
+            return _reranker_session, _reranker_tokenizer
+        model_path = os.path.join(RERANKER_DIR, "model.onnx")
+        tok_path = os.path.join(RERANKER_DIR, "tokenizer.json")
+        if not os.path.exists(model_path) or not os.path.exists(tok_path):
+            return None, None
+        import onnxruntime as ort
+        from tokenizers import Tokenizer
+        _reranker_session = ort.InferenceSession(
+            model_path, providers=["CPUExecutionProvider"],
+        )
+        _reranker_tokenizer = Tokenizer.from_file(tok_path)
+        _reranker_tokenizer.enable_truncation(max_length=512)
+        _reranker_tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=None)
+        return _reranker_session, _reranker_tokenizer
+
+
+def rerank(query, documents, limit=10):
+    """Score query-document pairs with the cross-encoder and return reranked indices.
+
+    Args:
+        query: search query string
+        documents: list of document texts to score against the query
+        limit: max results to return
+
+    Returns: list of (original_index, score) sorted by score descending.
+    """
+    session, tokenizer = _get_reranker()
+    if session is None:
+        return [(i, 0.0) for i in range(min(limit, len(documents)))]
+
+    # Cross-encoder takes (query, document) pairs — encode as pair sequences
+    pairs = [[query, doc] for doc in documents]
+    encodings = tokenizer.encode_batch(pairs)
+
+    input_ids = np.array([e.ids for e in encodings], dtype=np.int64)
+    attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64)
+    token_type_ids = np.array([e.type_ids for e in encodings], dtype=np.int64)
+
+    outputs = session.run(
+        None,
+        {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "token_type_ids": token_type_ids,
+        },
+    )
+    # Output is logits — higher = more relevant
+    scores = outputs[0].flatten()
+    ranked = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)
+    return [(i, float(scores[i])) for i in ranked[:limit]]
+
+
+# ---------------------------------------------------------------------------
+# Text chunking
+# ---------------------------------------------------------------------------
+
+_SENTENCE_RE = re.compile(r'(?<=[.!?])\s+')
+
+
+def chunk_text(title, body):
+    """Split body into chunks, each prefixed with title for context.
+
+    Strategy: split on double newlines (paragraphs). If a paragraph exceeds
+    MAX_TOKENS words, split at sentence boundaries. Each chunk is prefixed
+    with the page title.
+    """
+    if not body or not body.strip():
+        return [f"{title}"] if title else []
+
+    prefix = f"{title}: " if title else ""
+    # Rough word budget for chunk body (leave room for prefix)
+    prefix_words = len(prefix.split())
+    max_words = MAX_TOKENS - prefix_words  # approximate; tokenizer may differ
+
+    paragraphs = re.split(r'\n\s*\n', body.strip())
+    chunks = []
+
+    for para in paragraphs:
+        para = para.strip()
+        if len(para) < 20:
+            continue
+        words = para.split()
+        if len(words) <= max_words:
+            chunks.append(prefix + para)
+        else:
+            # Split paragraph into sentences
+            sentences = _SENTENCE_RE.split(para)
+            current = []
+            current_len = 0
+            for sent in sentences:
+                sent_words = len(sent.split())
+                if current_len + sent_words > max_words and current:
+                    chunks.append(prefix + " ".join(current))
+                    current = []
+                    current_len = 0
+                if sent_words > max_words:
+                    # Sentence too long — use sliding window
+                    s_words = sent.split()
+                    for i in range(0, len(s_words), max_words - 50):
+                        window = s_words[i:i + max_words]
+                        chunks.append(prefix + " ".join(window))
+                else:
+                    current.append(sent)
+                    current_len += sent_words
+            if current:
+                chunks.append(prefix + " ".join(current))
+
+    if not chunks and title:
+        chunks = [title]
+
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# Embedding
+# ---------------------------------------------------------------------------
+
+def embed(texts, is_query=False):
+    """Encode texts into L2-normalized float32 embeddings (N, 384).
+
+    For queries, prepend the model's query prefix.
+    Processes in batches of 32 to limit memory usage.
+    """
+    if not texts:
+        return np.empty((0, DIMS), dtype=np.float32)
+
+    session, tokenizer = _get_session()
+
+    if is_query:
+        texts = [QUERY_PREFIX + t for t in texts]
+
+    batch_size = 32
+    all_embeddings = []
+
+    for start in range(0, len(texts), batch_size):
+        batch = texts[start:start + batch_size]
+        encodings = tokenizer.encode_batch(batch)
+        input_ids = np.array([e.ids for e in encodings], dtype=np.int64)
+        attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64)
+        token_type_ids = np.zeros_like(input_ids)
+
+        outputs = session.run(
+            None,
+            {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            },
+        )
+        emb = outputs[0][:, 0, :]
+        all_embeddings.append(emb)
+
+    embeddings = np.concatenate(all_embeddings, axis=0)
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+    norms = np.maximum(norms, 1e-12)
+    embeddings = embeddings / norms
+    return _maybe_compress(embeddings.astype(np.float32))
+
+
+def _maybe_compress(embeddings):
+    """Compress embeddings to float16 if compression is enabled."""
+    try:
+        from db import get_setting
+        if get_setting("compress_embeddings", "0") == "1":
+            return embeddings.astype(np.float16)
+    except Exception:
+        pass
+    return embeddings
+
+
+def _decompress(embeddings):
+    """Decompress float16 embeddings to float32 if needed."""
+    if embeddings.dtype == np.float16:
+        return embeddings.astype(np.float32)
+    return embeddings
+
+
+def _blob_to_vec(buf):
+    """Decode a stored embedding blob to a float32 vector, inferring dtype from length."""
+    if len(buf) == DIMS * 2:
+        return np.frombuffer(buf, dtype=np.float16).astype(np.float32)
+    return np.frombuffer(buf, dtype=np.float32)
+
+
+# ---------------------------------------------------------------------------
+# HNSW index management
+# ---------------------------------------------------------------------------
+
+BATCH_SIZE = 50000
+
+def build_index(db=None):
+    """Load all embeddings from chunks table and build HNSW index in batches."""
+    import hnswlib
+    global _hnsw_index, _hnsw_ids
+
+    from db import get_db, return_db
+    own_db = db is None
+    if own_db:
+        db = get_db()
+
+    try:
+        total = db.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
+        if total == 0:
+            with _hnsw_lock:
+                _hnsw_index = None
+                _hnsw_ids = []
+            return
+
+        all_ids = []
+        all_embeddings = []
+
+        for offset in range(0, total, BATCH_SIZE):
+            rows = db.execute(
+                "SELECT id, embedding FROM chunks ORDER BY id LIMIT ? OFFSET ?",
+                (BATCH_SIZE, offset),
+            ).fetchall()
+            for r in rows:
+                emb = _blob_to_vec(r["embedding"])
+                all_ids.append(r["id"])
+                all_embeddings.append(emb)
+    finally:
+        if own_db:
+            return_db(db)
+
+    if not all_ids:
+        with _hnsw_lock:
+            _hnsw_index = None
+            _hnsw_ids = []
+        return
+
+    matrix = np.stack(all_embeddings)
+    n = len(all_ids)
+    ids = all_ids
+
+    index = hnswlib.Index(space="cosine", dim=DIMS)
+    index.init_index(max_elements=max(n, 1024), ef_construction=200, M=16)
+    index.add_items(matrix, list(range(n)))
+    index.set_ef(50)
+
+    with _hnsw_lock:
+        _hnsw_index = index
+        _hnsw_ids = ids
+
+
+def _add_to_index(chunk_ids, embeddings_matrix):
+    """Add new embeddings to the live HNSW index."""
+    import hnswlib
+    global _hnsw_index, _hnsw_ids
+
+    with _hnsw_lock:
+        if _hnsw_index is None:
+            index = hnswlib.Index(space="cosine", dim=DIMS)
+            index.init_index(max_elements=1024, ef_construction=200, M=16)
+            index.set_ef(50)
+            _hnsw_index = index
+            _hnsw_ids = []
+
+        current_max = _hnsw_index.get_max_elements()
+        needed = len(_hnsw_ids) + len(chunk_ids)
+        if needed > current_max:
+            _hnsw_index.resize_index(max(needed * 2, current_max * 2))
+
+        labels = list(range(len(_hnsw_ids), len(_hnsw_ids) + len(chunk_ids)))
+        _hnsw_index.add_items(embeddings_matrix, labels)
+        _hnsw_ids.extend(chunk_ids)
+
+
+# ---------------------------------------------------------------------------
+# Store embeddings for pages
+# ---------------------------------------------------------------------------
+
+def store_embeddings(page_id, title, body, db):
+    """Chunk, embed, and store embeddings for a page. Adds to HNSW index."""
+    chunks = chunk_text(title, body)
+    if not chunks:
+        return
+
+    embeddings_matrix = embed(chunks)
+    embeddings_matrix = _decompress(embeddings_matrix)
+
+    db.execute("DELETE FROM chunks WHERE page_id = ?", (page_id,))
+
+    new_ids = []
+    for i, (text, emb) in enumerate(zip(chunks, embeddings_matrix)):
+        cursor = db.execute(
+            "INSERT INTO chunks (page_id, remote_page_id, chunk_index, chunk_text, embedding) "
+            "VALUES (?, NULL, ?, ?, ?)",
+            (page_id, i, text, emb.tobytes()),
+        )
+        new_ids.append(cursor.lastrowid)
+    db.commit()
+
+    _add_to_index(new_ids, embeddings_matrix)
+
+
+def store_remote_embeddings(remote_page_id, title, note, db):
+    """Store a single embedding for a remote page (title + note)."""
+    text = f"{title}: {note}" if note else (title or "")
+    if not text.strip():
+        return
+
+    embeddings_matrix = embed([text])
+    embeddings_matrix = _decompress(embeddings_matrix)
+
+    db.execute("DELETE FROM chunks WHERE remote_page_id = ?", (remote_page_id,))
+    cursor = db.execute(
+        "INSERT INTO chunks (page_id, remote_page_id, chunk_index, chunk_text, embedding) "
+        "VALUES (NULL, ?, 0, ?, ?)",
+        (remote_page_id, text, embeddings_matrix[0].tobytes()),
+    )
+    db.commit()
+
+    _add_to_index([cursor.lastrowid], embeddings_matrix)
+
+
+# ---------------------------------------------------------------------------
+# Search
+# ---------------------------------------------------------------------------
+
+def semantic_search(query_text, limit=100, db=None):
+    """Search for pages by semantic similarity.
+
+    Returns: [(page_id, score, best_chunk_text), ...] sorted by score desc.
+    Groups by page_id, taking the max chunk score per page.
+    """
+    if _hnsw_index is None or not _hnsw_ids:
+        return []
+
+    query_emb = embed([query_text], is_query=True)
+
+    with _hnsw_lock:
+        if _hnsw_index is None or not _hnsw_ids:
+            return []
+        k = min(limit * 3, len(_hnsw_ids))  # oversample to account for grouping
+        if k == 0:
+            return []
+        labels, distances = _hnsw_index.knn_query(query_emb, k=k)
+
+    # Map HNSW labels back to chunk IDs
+    chunk_ids = [_hnsw_ids[int(lbl)] for lbl in labels[0]]
+    # cosine distance -> similarity: hnswlib returns 1-cosine for "cosine" space
+    scores = [1.0 - float(d) for d in distances[0]]
+
+    # Fetch chunk details from DB
+    from db import get_db, return_db
+    own_db = db is None
+    if own_db:
+        db = get_db()
+    try:
+        placeholders = ",".join("?" * len(chunk_ids))
+        rows = db.execute(
+            f"SELECT id, page_id, chunk_text FROM chunks WHERE id IN ({placeholders})",
+            chunk_ids,
+        ).fetchall()
+    finally:
+        if own_db:
+            return_db(db)
+
+    chunk_map = {r["id"]: r for r in rows}
+
+    # Group by page_id, keep best score and chunk text per page
+    page_best = {}  # page_id -> (score, chunk_text)
+    for cid, score in zip(chunk_ids, scores):
+        chunk = chunk_map.get(cid)
+        if not chunk or chunk["page_id"] is None:
+            continue
+        pid = chunk["page_id"]
+        if pid not in page_best or score > page_best[pid][0]:
+            page_best[pid] = (score, chunk["chunk_text"])
+
+    results = [(pid, score, text) for pid, (score, text) in page_best.items()]
+    results.sort(key=lambda x: x[1], reverse=True)
+    return results[:limit]
+
+
+def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=False):
+    """Merge BM25 and semantic results via RRF, optionally rerank with cross-encoder.
+
+    Default (two-stage): BM25 + semantic fused via RRF.
+    With use_reranker=True (three-stage): rerank top 20 with cross-encoder.
+
+    Returns: [(page_id, best_chunk_text), ...] in ranked order.
+    """
+    k = 60  # RRF constant
+
+    sem_results = semantic_search(query_text, limit=100, db=db)
+
+    best_chunks = {}  # page_id -> chunk_text
+    for _rank, (pid, _score, chunk_text) in enumerate(sem_results):
+        if pid not in best_chunks:
+            best_chunks[pid] = chunk_text
+
+    # When BM25 has no hits, use raw semantic similarity scores directly
+    # (RRF rank positions distort nearly-equal scores)
+    if not bm25_ranked_ids:
+        fused_ids = [(pid, score) for pid, score, _ in sem_results]
+    else:
+        rrf_scores = {}
+        for rank, pid in enumerate(bm25_ranked_ids):
+            rrf_scores[pid] = rrf_scores.get(pid, 0) + 1.0 / (k + rank + 1)
+        for rank, (pid, _score, chunk_text) in enumerate(sem_results):
+            rrf_scores[pid] = rrf_scores.get(pid, 0) + 1.0 / (k + rank + 1)
+        fused_ids = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)
+
+    fused = fused_ids
+    all_ids = [pid for pid, _ in fused]
+
+    if not all_ids:
+        return []
+
+    if not use_reranker:
+        return [(pid, best_chunks.get(pid, "")) for pid in all_ids[:limit]]
+
+    # --- Rerank top 20, append next 10 from RRF order ---
+    rerank_ids = all_ids[:20]
+    tail_ids = all_ids[20:30]
+
+    from db import get_db, return_db
+    own_db = db is None
+    if own_db:
+        db = get_db()
+    try:
+        placeholders = ",".join("?" * len(rerank_ids))
+        rows = db.execute(
+            f"SELECT id, title, body FROM pages WHERE id IN ({placeholders})",
+            rerank_ids,
+        ).fetchall()
+    finally:
+        if own_db:
+            return_db(db)
+
+    page_map = {r["id"]: r for r in rows}
+
+    doc_texts = []
+    ordered_ids = []
+    for pid in rerank_ids:
+        page = page_map.get(pid)
+        if not page:
+            continue
+        chunk = best_chunks.get(pid, "")
+        body_preview = chunk[:200] if chunk else page["body"][:200]
+        doc = f"{page['title']}. {body_preview}"
+        doc_texts.append(doc)
+        ordered_ids.append(pid)
+
+    if not doc_texts:
+        return []
+
+    try:
+        reranked = rerank(query_text, doc_texts, limit=20)
+        results = [(ordered_ids[idx], best_chunks.get(ordered_ids[idx], "")) for idx, _score in reranked]
+    except Exception:
+        results = [(pid, best_chunks.get(pid, "")) for pid in ordered_ids[:20]]
+
+    # Append next 10 from RRF order (no reranking)
+    reranked_set = {pid for pid, _ in results}
+    for pid in tail_ids:
+        if pid not in reranked_set:
+            results.append((pid, best_chunks.get(pid, "")))
+
+    return results[:30]
+
+
+# ---------------------------------------------------------------------------
+# Reindex
+# ---------------------------------------------------------------------------
+
+def reindex_all(db=None, progress_callback=None):
+    """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index."""
+    from db import get_db, return_db
+    own_db = db is None
+    if own_db:
+        db = get_db()
+    try:
+        # Clear existing chunks so everything is regenerated
+        db.execute("DELETE FROM chunks")
+        db.commit()
+
+        rows = db.execute(
+            "SELECT p.id, p.title, p.body, p.summary FROM pages p"
+        ).fetchall()
+
+        total = len(rows)
+        for i, row in enumerate(rows):
+            store_embeddings(row["id"], row["title"], row["body"], db)
+            if progress_callback:
+                progress_callback(i + 1, total)
+
+        # Also handle remote pages
+        remote_rows = db.execute(
+            "SELECT rp.id, rp.title, rp.note FROM remote_pages rp"
+        ).fetchall()
+
+        for rp in remote_rows:
+            store_remote_embeddings(rp["id"], rp["title"], rp["note"], db)
+    finally:
+        if own_db:
+            return_db(db)
+
+    build_index(db)

entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+# Generate Reticulum config with optional TCP peer
+# Set RNS_TCP_HOST and RNS_TCP_PORT env vars to connect to a remote instance
+
+CONFIG_DIR="/data/.reticulum"
+CONFIG_FILE="$CONFIG_DIR/config"
+
+mkdir -p "$CONFIG_DIR"
+
+if [ ! -f "$CONFIG_FILE" ]; then
+  cat > "$CONFIG_FILE" <<EOF
+[reticulum]
+  enable_transport = False
+  share_instance = No
+
+[logging]
+  loglevel = 4
+
+[interfaces]
+  [[Default Interface]]
+    type = AutoInterface
+    enabled = Yes
+
+  [[TCP Transport]]
+    type = TCPClientInterface
+    enabled = yes
+    target_host = ${RNS_TCP_HOST:-reticulum.derickphan.com}
+    target_port = ${RNS_TCP_PORT:-4242}
+EOF
+fi
+
+export RNS_CONFIG_DIR="$CONFIG_DIR"
+# Bind to 0.0.0.0 inside the container; isolation is handled by Docker's port mapping.
+exec python app.py --bind 0.0.0.0 "$@"

gateway.py

@@ -0,0 +1,194 @@
+import re
+import sys
+import time
+import threading
+import RNS
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from urllib.parse import parse_qs, urlparse
+
+APP_NAME = "tinyweb"
+ASPECTS = ["server"]
+GATEWAY_PORT = 8080
+REQUEST_TIMEOUT = 60
+MAX_BODY_SIZE = 16 * 1024 * 1024  # 16 MiB — covers /import and every other form
+
+
+class GatewayState:
+    reticulum = None
+    destination = None
+    link = None
+    link_lock = threading.Lock()
+    local_dispatch = None  # set when running inside app.py
+
+
+def resolve_destination(dest_hash_hex):
+    dest_hash = bytes.fromhex(dest_hash_hex)
+
+    if not RNS.Transport.has_path(dest_hash):
+        RNS.Transport.request_path(dest_hash)
+        print(f"Requesting path to {RNS.prettyhexrep(dest_hash)}...")
+        elapsed = 0
+        while not RNS.Transport.has_path(dest_hash) and elapsed < 15:
+            time.sleep(0.5)
+            elapsed += 0.5
+        if not RNS.Transport.has_path(dest_hash):
+            raise ConnectionError(f"Could not find path to {RNS.prettyhexrep(dest_hash)}")
+
+    server_identity = RNS.Identity.recall(dest_hash)
+    GatewayState.destination = RNS.Destination(
+        server_identity,
+        RNS.Destination.OUT,
+        RNS.Destination.SINGLE,
+        APP_NAME,
+        *ASPECTS,
+    )
+    print(f"Resolved destination: {RNS.prettyhexrep(dest_hash)}")
+
+
+def ensure_link():
+    with GatewayState.link_lock:
+        if GatewayState.link and GatewayState.link.status == RNS.Link.ACTIVE:
+            return GatewayState.link
+
+        print("Establishing link...")
+        link = RNS.Link(GatewayState.destination)
+        elapsed = 0
+        while link.status == RNS.Link.PENDING and elapsed < 15:
+            time.sleep(0.25)
+            elapsed += 0.25
+
+        if link.status != RNS.Link.ACTIVE:
+            raise ConnectionError("Link establishment failed")
+
+        GatewayState.link = link
+        print("Link established")
+        return link
+
+
+class GatewayHandler(BaseHTTPRequestHandler):
+
+    def _forward(self, method):
+        parsed = urlparse(self.path)
+        query = parse_qs(parsed.query)
+
+        body = {}
+        if method == "POST":
+            try:
+                length = int(self.headers.get("Content-Length", 0))
+            except ValueError:
+                self.send_error(400, "Invalid Content-Length")
+                return
+            if length < 0:
+                self.send_error(400, "Invalid Content-Length")
+                return
+            if length > MAX_BODY_SIZE:
+                self.send_error(413, "Request body too large")
+                return
+            raw = self.rfile.read(length).decode("utf-8", errors="replace")
+            body = parse_qs(raw)
+
+        # Parse cookies
+        cookies = {}
+        cookie_header = self.headers.get("Cookie", "")
+        if cookie_header:
+            for part in cookie_header.split(";"):
+                part = part.strip()
+                if "=" in part:
+                    k, v = part.split("=", 1)
+                    cookies[k.strip()] = v.strip()
+
+        request_data = {
+            "method": method,
+            "path": parsed.path,
+            "query": query,
+            "body": body,
+            "cookies": cookies,
+            "gateway_host": self.headers.get("Host", f"localhost:{GATEWAY_PORT}"),
+        }
+
+        try:
+            if GatewayState.local_dispatch:
+                resp = GatewayState.local_dispatch(request_data)
+            else:
+                link = ensure_link()
+                receipt = link.request(
+                    "/tinyweb",
+                    data=request_data,
+                    timeout=REQUEST_TIMEOUT,
+                )
+
+                # Wait for the response
+                elapsed = 0
+                done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED)
+                while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT:
+                    time.sleep(0.1)
+                    elapsed += 0.1
+
+                if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED):
+                    resp = receipt.get_response()
+                elif receipt.get_status() == RNS.RequestReceipt.FAILED:
+                    self.send_error(504, "Request to TinyWeb server failed")
+                    return
+                else:
+                    self.send_error(504, "Request to TinyWeb server timed out")
+                    return
+
+            self.send_response(resp["status"])
+            self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8"))
+            self.send_header("Referrer-Policy", "no-referrer")
+            self.send_header("X-Content-Type-Options", "nosniff")
+            self.send_header("X-Frame-Options", "DENY")
+            self.send_header("Content-Security-Policy",
+                             "default-src 'self'; "
+                             "style-src 'self' 'unsafe-inline'; "
+                             "script-src 'self' 'unsafe-inline'; "
+                             "img-src 'self' data:")
+            for k, v in resp.get("headers", {}).items():
+                self.send_header(k, v)
+            self.end_headers()
+            resp_body = resp.get("body", "")
+            if resp_body:
+                self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body)
+
+        except ConnectionError as e:
+            GatewayState.link = None
+            self.send_error(502, f"Gateway error: {e}")
+        except Exception as e:
+            GatewayState.link = None
+            self.send_error(502, f"Gateway error: {e}")
+
+    def do_GET(self):
+        self._forward("GET")
+
+    def do_POST(self):
+        self._forward("POST")
+
+    def log_message(self, format, *args):
+        try:
+            msg = format % args
+        except TypeError:
+            msg = format
+        # /bookmark carries a long-lived token and the URL being indexed —
+        # redact the query so it doesn't end up in stdout, journald, docker logs, etc.
+        msg = re.sub(r'(/bookmark)\?\S*', r'\1?[redacted]', msg)
+        print(f"[Gateway] {msg}")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(f"Usage: python gateway.py <destination_hash>")
+        print(f"  The destination hash is printed by app.py on startup.")
+        sys.exit(1)
+
+    dest_hash = sys.argv[1].replace("<", "").replace(">", "")
+
+    GatewayState.reticulum = RNS.Reticulum()
+    resolve_destination(dest_hash)
+
+    print(f"Gateway listening on http://localhost:{GATEWAY_PORT}")
+    print(f"Open http://localhost:{GATEWAY_PORT} in your browser")
+    HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler).serve_forever()
+
+
+if __name__ == "__main__":
+    main()

pyinstaller.spec

@@ -0,0 +1,81 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+import os
+import sys
+
+block_cipher = None
+
+# Hidden imports that PyInstaller can't detect automatically
+hiddenimports = [
+    "RNS",
+    "RNS.Destination",
+    "RNS.Identity",
+    "RNS.Reticulum",
+    "onnxruntime",
+    "onnxruntime.capi.onnxruntime_pybind11_state",
+    "tokenizers",
+    "huggingface_hub",
+    "hnswlib",
+    "bs4",
+    "beautifulsoup4",
+    "numpy",
+    "requests",
+]
+
+# Data files to include
+datas = [
+    ("themes", "themes"),
+]
+
+# Exclude unnecessary modules
+excludes = [
+    "test",
+    "tests",
+    "tkinter",
+    "matplotlib",
+    "scipy",
+    "pandas",
+    "IPython",
+    "jupyter",
+    "notebook",
+]
+
+a = Analysis(
+    ["app.py"],
+    pathex=[],
+    binaries=[],
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=excludes,
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    [],
+    name="TinyWeb",
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+filterwarnings =
+    ignore::DeprecationWarning

requirements-dev.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest

requirements.txt

@@ -1,2 +1,9 @@
+# forum: pip install tinyweb-forum (optional, adds URL discussion board)
 requests
 beautifulsoup4
+rns
+onnxruntime
+tokenizers
+hnswlib
+numpy
+huggingface_hub

rns_client.py

@@ -0,0 +1,110 @@
+import json
+import time
+import RNS
+
+APP_NAME = "tinyweb"
+ASPECTS = ["server"]
+
+# Two-tier timeout profiles: fast first, then slow for LoRa/multi-hop links
+_TIMEOUT_TIERS = [
+    {"path": 15, "link": 15, "request": 30, "poll": 0.25},
+    {"path": 60, "link": 60, "request": 120, "poll": 1.0},
+]
+
+
+def fetch_remote_sites(dest_hash_hex, since=""):
+    """
+    Connect to a remote TinyWeb instance over Reticulum and fetch its
+    shared sites. Returns the response dict from /api/sites, or raises
+    an exception on failure. Pass `since` as ISO timestamp for delta sync.
+
+    Uses progressive timeouts: tries fast first, then retries with longer
+    timeouts for slow links (LoRa, multi-hop).
+    """
+    last_error = None
+    for tier in _TIMEOUT_TIERS:
+        try:
+            return _fetch(dest_hash_hex, since, tier)
+        except PermissionError:
+            raise  # Don't retry permission errors
+        except Exception as e:
+            last_error = e
+            continue
+    raise ConnectionError(
+        f"Could not reach {dest_hash_hex} after {len(_TIMEOUT_TIERS)} attempts: {last_error}"
+    )
+
+
+def _fetch(dest_hash_hex, since, timeouts):
+    """Single fetch attempt with the given timeout profile."""
+    dest_hash = bytes.fromhex(dest_hash_hex)
+    poll = timeouts["poll"]
+
+    # Resolve path if needed
+    if not RNS.Transport.has_path(dest_hash):
+        RNS.Transport.request_path(dest_hash)
+        elapsed = 0
+        while not RNS.Transport.has_path(dest_hash) and elapsed < timeouts["path"]:
+            time.sleep(poll)
+            elapsed += poll
+        if not RNS.Transport.has_path(dest_hash):
+            raise ConnectionError(
+                f"Could not find path to {dest_hash_hex} ({timeouts['path']}s timeout)"
+            )
+
+    server_identity = RNS.Identity.recall(dest_hash)
+    if server_identity is None:
+        raise ConnectionError(f"Could not recall identity for {dest_hash_hex}")
+
+    destination = RNS.Destination(
+        server_identity,
+        RNS.Destination.OUT,
+        RNS.Destination.SINGLE,
+        APP_NAME,
+        *ASPECTS,
+    )
+
+    # Establish link
+    link = RNS.Link(destination)
+    elapsed = 0
+    while link.status == RNS.Link.PENDING and elapsed < timeouts["link"]:
+        time.sleep(poll)
+        elapsed += poll
+
+    if link.status != RNS.Link.ACTIVE:
+        raise ConnectionError(
+            f"Could not establish link to {dest_hash_hex} ({timeouts['link']}s timeout)"
+        )
+
+    try:
+        query = {"since": [since]} if since else {}
+        request_data = {
+            "method": "GET",
+            "path": "/api/sites",
+            "query": query,
+            "body": {},
+            "gateway_host": "",
+        }
+
+        req_timeout = timeouts["request"]
+        receipt = link.request("/tinyweb", data=request_data, timeout=req_timeout)
+
+        elapsed = 0
+        done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED)
+        while receipt.get_status() not in done and elapsed < req_timeout:
+            time.sleep(poll)
+            elapsed += poll
+
+        if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED):
+            resp = receipt.get_response()
+            if resp["status"] == 403:
+                raise PermissionError("That instance has sharing disabled.")
+            if resp["status"] != 200:
+                raise ConnectionError(f"Remote returned status {resp['status']}")
+            return json.loads(resp["body"])
+        else:
+            raise ConnectionError(
+                f"Request failed or timed out ({req_timeout}s timeout)"
+            )
+    finally:
+        link.teardown()

start.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /nix/store/vhgmnrmvvfdiw0kc2xz8px7rvg60lszc-python3-3.13.12-env/bin/python /home/lichenblankie/apps/tinyweb/app.py --bind 0.0.0.0

templates.py

@@ -0,0 +1,37 @@
+import html
+from db import get_setting
+
+FORUM_ENABLED = False
+
+def esc(s):
+    return html.escape(str(s))
+
+
+DEFAULT_TEMPLATE = "<html>\n<head>\n<meta name=\"referrer\" content=\"no-referrer\">\n<meta http-equiv=\"x-dns-prefetch-control\" content=\"off\">\n</head>\n<body>\n{{content}}\n</body>\n</html>"
+
+
+def _default_template():
+    name = esc(get_setting("site_name", "tinyweb"))
+    forum_link = ' | <a href="/forum">forum</a>' if FORUM_ENABLED else ""
+    return (
+        '<html>\n<head>\n<meta name="referrer" content="no-referrer">\n<meta http-equiv="x-dns-prefetch-control" content="off">\n</head>\n<body>\n'
+        f'<p><b><a href="/">{name}</a></b>'
+        ' | <a href="/">search</a> | <a href="/pages">browse</a>'
+        ' | <a href="/tags">tags</a> | <a href="/subscriptions">subscriptions</a>'
+        f'{forum_link}'
+        ' | <a href="/style">customize</a> | <a href="/about">about</a></p>\n'
+        "<hr>\n{{content}}\n</body>\n</html>"
+    )
+
+
+def wrap_page(body_html, use_default=False):
+    if use_default:
+        template = _default_template()
+    else:
+        template = get_setting("custom_template") or _default_template()
+        if "{{content}}" not in template:
+            template = _default_template()
+    forum_link = ' <a href="/forum">forum</a>' if FORUM_ENABLED else ""
+    template = template.replace("{{forum_link}}", forum_link)
+    template = template.replace("{{site_name}}", esc(get_setting("site_name", "tinyweb")))
+    return template.replace("{{content}}", body_html)

tests/test_csrf.py

@@ -0,0 +1,60 @@
+"""Tests for `_check_csrf` — form-submission CSRF protection.
+
+Every POST handler calls this to verify the submitted _csrf field matches
+the token stored in the thread-local (which is seeded from the cookie by
+`dispatch_request`). Missing or mismatched tokens must fail closed.
+"""
+import handlers as handlers_module
+from handlers import _check_csrf, _csrf_field, _get_csrf_token
+
+
+def _set_token(token):
+    handlers_module._request_local.csrf_token = token
+
+
+def _clear_token():
+    if hasattr(handlers_module._request_local, "csrf_token"):
+        del handlers_module._request_local.csrf_token
+
+
+def teardown_function(_):
+    _clear_token()
+
+
+def test_rejects_missing_token_in_body():
+    _set_token("server-side-token")
+    assert _check_csrf({}) is False
+
+
+def test_rejects_empty_token_in_body():
+    _set_token("server-side-token")
+    assert _check_csrf({"_csrf": [""]}) is False
+
+
+def test_rejects_mismatched_token():
+    _set_token("server-side-token")
+    assert _check_csrf({"_csrf": ["attacker-token"]}) is False
+
+
+def test_accepts_matching_token():
+    _set_token("server-side-token")
+    assert _check_csrf({"_csrf": ["server-side-token"]}) is True
+
+
+def test_rejects_when_server_token_missing():
+    """If the server-side token is empty (shouldn't happen after dispatch_request
+    seeds it, but be defensive), the check must fail closed."""
+    _clear_token()
+    assert _check_csrf({"_csrf": ["anything"]}) is False
+
+
+def test_csrf_field_renders_current_token():
+    _set_token("abc123")
+    field = _csrf_field()
+    assert 'name="_csrf"' in field
+    assert 'value="abc123"' in field
+
+
+def test_get_csrf_token_returns_empty_when_unset():
+    _clear_token()
+    assert _get_csrf_token() == ""

tests/test_db_index_url.py

@@ -0,0 +1,155 @@
+"""Tests for `index_url` — the main write path.
+
+Covers UPSERT behavior, links being replaced on re-index, FTS index staying
+in sync via triggers, and the connection pool returning clean connections.
+"""
+from unittest.mock import patch
+
+from conftest import patch_dns_ok
+import db as db_module
+from db import get_db, return_db, index_url
+
+
+def _mock_fetch_page(title="Test Page", body="test body text", links=None, meta=""):
+    """Return a replacement for db.fetch_page that yields canned data."""
+    links = links or []
+    def fake(url):
+        return (title, body, links, meta)
+    return fake
+
+
+def test_insert_creates_page_row_and_fts_entry(temp_db, monkeypatch):
+    patch_dns_ok(monkeypatch)
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="Rust Intro", body="ownership and borrowing basics", links=[],
+    ))
+    index_url("https://example.com/rust")
+
+    db = get_db()
+    try:
+        row = db.execute("SELECT id, title, body FROM pages").fetchone()
+        assert row is not None
+        assert row["title"] == "Rust Intro"
+        assert "ownership" in row["body"]
+        # Verify FTS trigger fired.
+        fts_hits = db.execute(
+            "SELECT rowid FROM pages_fts WHERE pages_fts MATCH 'ownership*'"
+        ).fetchall()
+        assert len(fts_hits) == 1
+        assert fts_hits[0]["rowid"] == row["id"]
+    finally:
+        return_db(db)
+
+
+def test_re_indexing_same_url_updates_in_place(temp_db, monkeypatch):
+    patch_dns_ok(monkeypatch)
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="First Title", body="first body", links=[],
+    ))
+    index_url("https://example.com/page")
+
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="Second Title", body="second body", links=[],
+    ))
+    index_url("https://example.com/page")
+
+    db = get_db()
+    try:
+        rows = db.execute("SELECT title, body FROM pages").fetchall()
+    finally:
+        return_db(db)
+    assert len(rows) == 1, "re-indexing should UPDATE not INSERT"
+    assert rows[0]["title"] == "Second Title"
+
+
+def test_links_replaced_on_reindex(temp_db, monkeypatch):
+    patch_dns_ok(monkeypatch)
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="T", body="b",
+        links=[("https://example.com/a", "first"), ("https://example.com/b", "second")],
+    ))
+    index_url("https://example.com/src")
+
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="T", body="b",
+        links=[("https://example.com/c", "third-only")],
+    ))
+    index_url("https://example.com/src")
+
+    db = get_db()
+    try:
+        rows = db.execute("SELECT url FROM links").fetchall()
+    finally:
+        return_db(db)
+    urls = {r["url"] for r in rows}
+    assert urls == {"https://example.com/c"}, "old links should be deleted on reindex"
+
+
+def test_url_cleaned_before_insert(temp_db, monkeypatch):
+    """index_url should apply clean_url before touching the DB, so tracking params
+    don't create duplicate rows."""
+    patch_dns_ok(monkeypatch)
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(title="T", body="b"))
+    index_url("https://example.com/page?utm_source=twitter#frag")
+
+    db = get_db()
+    try:
+        rows = db.execute("SELECT url FROM pages").fetchall()
+    finally:
+        return_db(db)
+    assert len(rows) == 1
+    assert rows[0]["url"] == "https://example.com/page"
+
+
+def test_summary_populated_from_meta_description(temp_db, monkeypatch):
+    patch_dns_ok(monkeypatch)
+    long_meta = "A thoughtful description that exceeds twenty chars"
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="T", body="b", meta=long_meta,
+    ))
+    index_url("https://example.com/page")
+
+    db = get_db()
+    try:
+        row = db.execute("SELECT summary FROM pages").fetchone()
+    finally:
+        return_db(db)
+    assert row["summary"] == long_meta
+
+
+def test_short_meta_description_not_stored_as_summary(temp_db, monkeypatch):
+    patch_dns_ok(monkeypatch)
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(
+        title="T", body="b", meta="too short",
+    ))
+    index_url("https://example.com/page")
+
+    db = get_db()
+    try:
+        row = db.execute("SELECT summary FROM pages").fetchone()
+    finally:
+        return_db(db)
+    assert row["summary"] == ""
+
+
+def test_pool_returns_clean_connection(temp_db, monkeypatch):
+    """Regression for 1bc695f — `return_db` should roll back uncommitted work
+    so the next consumer doesn't see stale state."""
+    patch_dns_ok(monkeypatch)
+    monkeypatch.setattr(db_module, "fetch_page", _mock_fetch_page(title="T", body="b"))
+    index_url("https://example.com/one")
+
+    # Take a connection, make a dirty uncommitted change, return it.
+    db = get_db()
+    db.execute("INSERT INTO pages (url, title, body) VALUES (?, ?, ?)",
+               ("https://dirty.example.com/", "dirty", "dirty"))
+    # NOTE: no commit here — this is the dirty state we want rolled back.
+    return_db(db)
+
+    # A later consumer must not see the dirty row.
+    db2 = get_db()
+    try:
+        urls = {r["url"] for r in db2.execute("SELECT url FROM pages").fetchall()}
+    finally:
+        return_db(db2)
+    assert "https://dirty.example.com/" not in urls

tests/test_db_schema.py

@@ -0,0 +1,90 @@
+"""Tests for `init_db` and the settings key-value store.
+
+`init_db` is called unconditionally on startup, so it must be idempotent
+and create every table/trigger the rest of the app expects.
+"""
+from db import get_db, return_db, init_db, get_setting, set_setting, get_site_name
+
+
+EXPECTED_TABLES = {
+    "pages", "links", "settings", "subscriptions",
+    "remote_pages", "tags", "page_tags", "chunks",
+    # FTS5 virtual tables:
+    "pages_fts", "remote_pages_fts",
+}
+
+
+def test_all_expected_tables_exist(temp_db):
+    db = get_db()
+    try:
+        rows = db.execute(
+            "SELECT name FROM sqlite_master WHERE type IN ('table') AND name NOT LIKE 'sqlite_%'"
+        ).fetchall()
+        names = {r["name"] for r in rows}
+    finally:
+        return_db(db)
+    missing = EXPECTED_TABLES - names
+    assert not missing, f"tables missing after init_db: {missing}"
+
+
+def test_fts_triggers_exist(temp_db):
+    db = get_db()
+    try:
+        rows = db.execute(
+            "SELECT name FROM sqlite_master WHERE type = 'trigger'"
+        ).fetchall()
+        names = {r["name"] for r in rows}
+    finally:
+        return_db(db)
+    # These triggers keep pages_fts in sync with pages on insert/update/delete.
+    for trigger in ("pages_ai", "pages_ad", "pages_au"):
+        assert trigger in names, f"missing trigger {trigger}"
+
+
+def test_init_db_is_idempotent(temp_db):
+    """Running init_db twice on the same DB must not error or duplicate anything."""
+    init_db()
+    init_db()  # second call should be a no-op
+    db = get_db()
+    try:
+        count = db.execute(
+            "SELECT count(*) FROM sqlite_master WHERE name = 'pages'"
+        ).fetchone()[0]
+    finally:
+        return_db(db)
+    assert count == 1
+
+
+def test_get_setting_returns_default_when_missing(temp_db):
+    assert get_setting("nonexistent", "fallback") == "fallback"
+    assert get_setting("nonexistent") == ""
+
+
+def test_set_setting_then_get(temp_db):
+    set_setting("site_name", "my-personal-index")
+    assert get_setting("site_name") == "my-personal-index"
+
+
+def test_set_setting_updates_existing(temp_db):
+    set_setting("key", "first")
+    set_setting("key", "second")
+    assert get_setting("key") == "second"
+
+
+def test_get_site_name_has_default(temp_db):
+    assert get_site_name() == "tinyweb"
+
+
+def test_get_site_name_reflects_override(temp_db):
+    set_setting("site_name", "custom-site")
+    assert get_site_name() == "custom-site"
+
+
+def test_foreign_keys_pragma_enabled(temp_db):
+    """Pool connections should have foreign_keys=ON so CASCADE deletes work."""
+    db = get_db()
+    try:
+        row = db.execute("PRAGMA foreign_keys").fetchone()
+    finally:
+        return_db(db)
+    assert row[0] == 1

tests/test_fts_sanitizer.py

@@ -0,0 +1,113 @@
+"""Tests for `_sanitize_fts_query`.
+
+The sanitizer is the boundary between user input and FTS5 MATCH syntax.
+Commit 1bc695f tightened it after noticing that colons and operator words
+could escape the quoting. These tests keep that regression dead.
+"""
+import pytest
+
+from handlers import _sanitize_fts_query
+
+
+def test_empty_query_returns_no_match_token():
+    assert _sanitize_fts_query("") == '""'
+    assert _sanitize_fts_query("   ") == '""'
+
+
+def test_single_word_becomes_prefix_match():
+    assert _sanitize_fts_query("rust") == "rust*"
+
+
+def test_multi_word_quotes_all_but_last():
+    result = _sanitize_fts_query("rust borrow checker")
+    assert result == '"rust" "borrow" checker*'
+
+
+def test_stopwords_are_dropped():
+    # "the" and "a" should vanish; only "cat" remains (and gets prefix star).
+    assert _sanitize_fts_query("the a cat") == "cat*"
+
+
+def test_all_stopwords_returns_no_match_token():
+    assert _sanitize_fts_query("the and or") == '""'
+
+
+@pytest.mark.parametrize("bad_char", ["'", "(", ")", "+", "-", "^", "~", ":"])
+def test_fts5_operators_stripped_from_tokens(bad_char):
+    """FTS5 special chars inside user tokens must not survive — regression for 1bc695f.
+
+    The sanitizer legitimately adds `"` around tokens and a trailing `*` for prefix
+    matching; both are excluded from this check.
+    """
+    payload = f"foo{bad_char}bar"
+    out = _sanitize_fts_query(payload)
+    assert bad_char not in out, f"{bad_char!r} leaked into {out!r}"
+
+
+def test_asterisk_only_appears_as_trailing_prefix():
+    """Input `*` should not become an in-token asterisk; the sanitizer's trailing `*` is fine."""
+    out = _sanitize_fts_query("foo*bar")
+    assert out.count("*") <= 1
+    if "*" in out:
+        assert out.endswith("*")
+
+
+def test_quote_in_input_does_not_break_out_of_quoted_token():
+    """A `"` in user input must not close the sanitizer's protective quoting.
+
+    The sanitizer wraps each non-last token in double quotes; if a stray `"` from
+    the user slipped through, the resulting FTS5 expression would be interpreted
+    as broken syntax or, worse, a column filter.
+    """
+    out = _sanitize_fts_query('foo"bar baz"qux')
+    # Each pair of quotes in the output should be balanced and around a clean token.
+    assert out.count('"') % 2 == 0
+    # No embedded quotes inside a quoted region.
+    import re
+    for match in re.findall(r'"[^"]*"', out):
+        inner = match[1:-1]
+        assert '"' not in inner
+
+
+@pytest.mark.parametrize("op", ["AND", "OR", "NOT", "NEAR", "and", "or", "not", "near"])
+def test_fts5_operator_words_dropped(op):
+    """AND/OR/NOT/NEAR would be interpreted as operators on the unquoted last token."""
+    out = _sanitize_fts_query(f"foo {op} bar")
+    # the operator word itself should not appear
+    assert op.upper() not in out.upper().split('"'), f"operator {op!r} survived in {out!r}"
+
+
+def test_injection_payload_produces_valid_fts5():
+    """End-to-end: a realistic injection payload must produce syntactically valid FTS5.
+
+    We run the sanitized output through a throwaway FTS5 table; if the sanitizer
+    leaks operator characters the MATCH either raises or interprets malicious syntax.
+    """
+    import sqlite3
+    conn = sqlite3.connect(":memory:")
+    conn.execute("CREATE VIRTUAL TABLE t USING fts5(body)")
+    conn.execute("INSERT INTO t (body) VALUES ('hello world')")
+
+    for payload in [
+        'foo": OR bar NOT baz AND qux*()',
+        '" OR 1=1 --',
+        "title:secret AND public",
+        "(((",
+        "^^^~~~",
+    ]:
+        q = _sanitize_fts_query(payload)
+        # Must not raise — if operators leaked, FTS5 would error or mis-parse.
+        conn.execute("SELECT * FROM t WHERE t MATCH ?", (q,)).fetchall()
+    conn.close()
+
+
+def test_whitespace_only_tokens_dropped():
+    # tokens that become empty after stripping special chars should not produce bare quotes
+    out = _sanitize_fts_query('""" "" ""')
+    assert out == '""'
+
+
+def test_colon_stripped():
+    """Regression for 1bc695f — colon is an FTS5 column filter and must be stripped."""
+    out = _sanitize_fts_query("title:secret")
+    assert ":" not in out

tests/test_gateway_limits.py

@@ -0,0 +1,164 @@
+"""Tests for gateway-level guards: body-size cap and Reticulum surface whitelist.
+
+Regression targets from commit 1bc695f — a 16 MiB upload limit (DoS guard)
+and a strict GET-/api/sites-only whitelist for requests arriving over the
+Reticulum mesh (CSRF can't protect mesh callers, so gate by whitelist).
+"""
+import io
+
+import pytest
+
+import app as app_module
+from gateway import GatewayHandler, MAX_BODY_SIZE
+
+
+class FakeHeaders:
+    """Minimal replacement for http.server request headers."""
+    def __init__(self, items=None):
+        self._items = dict(items or {})
+
+    def get(self, key, default=None):
+        return self._items.get(key, default)
+
+
+class FakeGatewayHandler(GatewayHandler):
+    """Bypass the socket-bound __init__ and capture response calls in memory."""
+    def __init__(self, path="/", method="POST", headers=None, rfile=None):
+        self.path = path
+        self.command = method
+        self.headers = FakeHeaders(headers or {})
+        self.rfile = rfile or io.BytesIO()
+        self.wfile = io.BytesIO()
+        self._captured = {
+            "error": None, "status": None, "headers": [], "body_written": None,
+        }
+
+    def send_error(self, code, msg=""):
+        self._captured["error"] = (code, msg)
+
+    def send_response(self, code):
+        self._captured["status"] = code
+
+    def send_header(self, k, v):
+        self._captured["headers"].append((k, v))
+
+    def end_headers(self):
+        pass
+
+
+def test_post_over_size_cap_rejected_with_413():
+    """Regression for 1bc695f: request bodies over MAX_BODY_SIZE must be rejected
+    without being read into memory."""
+    oversize = MAX_BODY_SIZE + 1
+    handler = FakeGatewayHandler(
+        path="/add",
+        method="POST",
+        headers={"Content-Length": str(oversize)},
+    )
+    handler._forward("POST")
+    assert handler._captured["error"] is not None
+    code, _msg = handler._captured["error"]
+    assert code == 413
+
+
+def test_post_at_size_cap_accepted():
+    """A body exactly at MAX_BODY_SIZE should not be rejected by the size check."""
+    handler = FakeGatewayHandler(
+        path="/_does_not_matter",
+        method="POST",
+        headers={"Content-Length": str(MAX_BODY_SIZE)},
+        # rfile has no data; handler will try to read; local_dispatch isn't set.
+        # We only care that the 413 check passes, not that the request succeeds.
+        rfile=io.BytesIO(b""),
+    )
+    # Stub out local_dispatch so _forward doesn't try the network path.
+    from gateway import GatewayState
+    original = GatewayState.local_dispatch
+    GatewayState.local_dispatch = lambda data: {
+        "status": 404, "content_type": "text/plain", "body": "nope",
+    }
+    try:
+        handler._forward("POST")
+    finally:
+        GatewayState.local_dispatch = original
+    # Not a 413, because the body is exactly at the cap (cap is inclusive).
+    if handler._captured["error"]:
+        assert handler._captured["error"][0] != 413
+
+
+def test_negative_content_length_rejected():
+    handler = FakeGatewayHandler(
+        path="/add",
+        method="POST",
+        headers={"Content-Length": "-1"},
+    )
+    handler._forward("POST")
+    assert handler._captured["error"] is not None
+    code, _msg = handler._captured["error"]
+    assert code == 400
+
+
+def test_invalid_content_length_rejected():
+    handler = FakeGatewayHandler(
+        path="/add",
+        method="POST",
+        headers={"Content-Length": "abc"},
+    )
+    handler._forward("POST")
+    assert handler._captured["error"] is not None
+    code, _msg = handler._captured["error"]
+    assert code == 400
+
+
+# -------- Reticulum mesh surface whitelist --------
+
+
+def test_mesh_rejects_non_api_sites_get():
+    """Regression for 1bc695f: remote mesh callers can only GET /api/sites."""
+    resp = app_module.rns_request_handler(
+        path="/tinyweb",
+        data={"method": "GET", "path": "/pages", "query": {}, "body": {}, "gateway_host": ""},
+        request_id="x", link_id="y", remote_identity=None, requested_at=0,
+    )
+    assert resp["status"] == 403
+
+
+def test_mesh_rejects_post_to_api_sites():
+    resp = app_module.rns_request_handler(
+        path="/tinyweb",
+        data={"method": "POST", "path": "/api/sites", "query": {}, "body": {}, "gateway_host": ""},
+        request_id="x", link_id="y", remote_identity=None, requested_at=0,
+    )
+    assert resp["status"] == 403
+
+
+def test_mesh_rejects_sensitive_local_endpoints():
+    for path in ("/add", "/delete/1", "/style", "/import", "/export"):
+        resp = app_module.rns_request_handler(
+            path="/tinyweb",
+            data={"method": "GET", "path": path, "query": {}, "body": {}, "gateway_host": ""},
+            request_id="x", link_id="y", remote_identity=None, requested_at=0,
+        )
+        assert resp["status"] == 403, f"path {path!r} leaked through mesh whitelist"
+
+
+def test_mesh_allows_api_sites_get(temp_db, csrf_session):
+    """Sanity check: the one whitelisted combination is accepted."""
+    resp = app_module.rns_request_handler(
+        path="/tinyweb",
+        data={"method": "GET", "path": "/api/sites", "query": {}, "body": {}, "gateway_host": ""},
+        request_id="x", link_id="y", remote_identity=None, requested_at=0,
+    )
+    # Status depends on handler output; 200 is the happy path.
+    assert resp["status"] in (200, 403)  # 403 if sharing is disabled by default
+
+
+def test_mesh_handles_missing_data_payload():
+    """Regression-minded check: a None or malformed data object shouldn't crash."""
+    resp = app_module.rns_request_handler(
+        path="/tinyweb",
+        data=None,
+        request_id="x", link_id="y", remote_identity=None, requested_at=0,
+    )
+    # Default data has method=GET, path=/ which is not in the whitelist.
+    assert resp["status"] == 403

tests/test_handlers_pages.py

@@ -0,0 +1,174 @@
+"""Tests for `handle_bulk_action`, edit flow, and the bulk-delete confirm step.
+
+The bulk-delete confirmation flow is a data-loss guard added in commit
+8dffd8c — a stray POST without `confirmed=1` must render the confirmation
+page instead of actually deleting.
+"""
+from db import get_db, return_db
+from handlers import (
+    handle_bulk_action,
+    handle_edit_form,
+    handle_edit_submit,
+    handle_pages,
+)
+
+
+def _all_urls(seeded_db):
+    db = get_db()
+    try:
+        return {r["url"] for r in db.execute("SELECT url FROM pages").fetchall()}
+    finally:
+        return_db(db)
+
+
+def _page_id(seeded_db, url):
+    db = get_db()
+    try:
+        return db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()["id"]
+    finally:
+        return_db(db)
+
+
+def test_bulk_delete_without_confirmed_renders_confirm_page(seeded_db, csrf_session):
+    """Regression for 8dffd8c: bulk delete must NOT delete until confirmed=1 is set."""
+    pid = _page_id(seeded_db, "https://example.com/rust-intro")
+    urls_before = _all_urls(seeded_db)
+
+    resp = handle_bulk_action({
+        "ids": [str(pid)],
+        "action": ["delete"],
+    })
+    assert resp["status"] == 200
+    assert "confirm delete" in resp["body"].lower()
+    assert "Rust Intro" in resp["body"]
+    # Must still show a hidden confirmed=1 field in the follow-up form.
+    assert 'name="confirmed" value="1"' in resp["body"]
+
+    # Crucially: nothing should have been deleted.
+    assert _all_urls(seeded_db) == urls_before
+
+
+def test_bulk_delete_with_confirmed_actually_deletes(seeded_db, csrf_session):
+    pid = _page_id(seeded_db, "https://example.com/rust-intro")
+
+    resp = handle_bulk_action({
+        "ids": [str(pid)],
+        "action": ["delete"],
+        "confirmed": ["1"],
+    })
+    # Confirmed delete redirects back to /pages.
+    assert resp["status"] in (302, 303)
+
+    urls = _all_urls(seeded_db)
+    assert "https://example.com/rust-intro" not in urls
+    # Other pages untouched.
+    assert "https://example.com/python-tips" in urls
+
+
+def test_bulk_delete_with_no_ids_redirects(seeded_db, csrf_session):
+    resp = handle_bulk_action({
+        "ids": [],
+        "action": ["delete"],
+        "confirmed": ["1"],
+    })
+    assert resp["status"] in (302, 303)
+    assert _all_urls(seeded_db) == {
+        "https://example.com/rust-intro",
+        "https://example.com/python-tips",
+        "https://example.com/ocaml-why",
+        "https://news.example.org/mesh",
+    }
+
+
+def test_bulk_delete_rejects_non_integer_ids(seeded_db, csrf_session):
+    resp = handle_bulk_action({
+        "ids": ["not-a-number"],
+        "action": ["delete"],
+        "confirmed": ["1"],
+    })
+    assert resp["status"] == 400
+
+
+def test_bulk_retag_add_mode_merges_tags(seeded_db, csrf_session):
+    pid = _page_id(seeded_db, "https://example.com/python-tips")
+
+    handle_bulk_action({
+        "ids": [str(pid)],
+        "action": ["retag"],
+        "bulk_tags": ["scripting, tutorials"],
+        "tag_mode": ["add"],
+    })
+    db = get_db()
+    try:
+        rows = db.execute(
+            "SELECT t.name FROM tags t JOIN page_tags pt ON pt.tag_id = t.id "
+            "WHERE pt.page_id = ? ORDER BY t.name",
+            (pid,),
+        ).fetchall()
+    finally:
+        return_db(db)
+    tags = [r["name"] for r in rows]
+    assert "python" in tags  # existing kept
+    assert "scripting" in tags  # new added
+    assert "tutorials" in tags
+
+
+def test_bulk_retag_replace_mode_overwrites_tags(seeded_db, csrf_session):
+    pid = _page_id(seeded_db, "https://example.com/python-tips")
+
+    handle_bulk_action({
+        "ids": [str(pid)],
+        "action": ["retag"],
+        "bulk_tags": ["one, two"],
+        "tag_mode": ["replace"],
+    })
+    db = get_db()
+    try:
+        rows = db.execute(
+            "SELECT t.name FROM tags t JOIN page_tags pt ON pt.tag_id = t.id "
+            "WHERE pt.page_id = ?",
+            (pid,),
+        ).fetchall()
+    finally:
+        return_db(db)
+    tags = {r["name"] for r in rows}
+    assert tags == {"one", "two"}
+    assert "python" not in tags
+
+
+def test_edit_form_renders_current_values(seeded_db, csrf_session):
+    pid = _page_id(seeded_db, "https://example.com/rust-intro")
+    resp = handle_edit_form(pid)
+    assert resp["status"] == 200
+    assert "Rust Intro" in resp["body"]
+    # Existing tags should appear in the tag field.
+    assert "rust" in resp["body"]
+
+
+def test_edit_form_404_for_unknown_page(temp_db, csrf_session):
+    resp = handle_edit_form(99999)
+    assert resp["status"] == 404
+
+
+def test_edit_submit_updates_title_and_note(seeded_db, csrf_session):
+    pid = _page_id(seeded_db, "https://example.com/rust-intro")
+    handle_edit_submit(pid, {
+        "title": ["New Rust Title"],
+        "note": ["new annotation"],
+        "tags": ["rust, updated"],
+    })
+    db = get_db()
+    try:
+        row = db.execute("SELECT title, note FROM pages WHERE id = ?", (pid,)).fetchone()
+    finally:
+        return_db(db)
+    assert row["title"] == "New Rust Title"
+    assert row["note"] == "new annotation"
+
+
+def test_handle_pages_lists_indexed_pages(seeded_db, csrf_session):
+    resp = handle_pages({})
+    assert resp["status"] == 200
+    # Every seeded page title appears on the list page.
+    for title in ("Rust Intro", "Python Tips", "Why OCaml", "Mesh Networking"):
+        assert title in resp["body"]

tests/test_handlers_search.py

@@ -0,0 +1,63 @@
+"""Tests for `handle_search` — the home page + primary user flow."""
+from handlers import handle_search
+
+
+def test_empty_index_empty_query_shows_welcome(temp_db, csrf_session):
+    resp = handle_search({})
+    assert resp["status"] == 200
+    body = resp["body"]
+    assert "Your index is empty" in body
+    # Links the welcome panel offers as equal-weight starting points.
+    assert "/add" in body
+    assert "/style" in body
+    assert "/subscriptions" in body
+
+
+def test_empty_index_with_query_shows_no_results(temp_db, csrf_session):
+    resp = handle_search({"q": ["rust"]})
+    assert resp["status"] == 200
+    assert "No results in your index" in resp["body"]
+
+
+def test_populated_index_with_matching_query_returns_results(seeded_db, csrf_session):
+    resp = handle_search({"q": ["rust"]})
+    assert resp["status"] == 200
+    assert "Rust Intro" in resp["body"]
+    # Page count shown in meta line.
+    assert "4 pages indexed" in resp["body"]
+
+
+def test_query_only_matches_relevant_pages(seeded_db, csrf_session):
+    resp = handle_search({"q": ["ocaml"]})
+    body = resp["body"]
+    assert "Why OCaml" in body
+    assert "Python Tips" not in body
+    assert "Rust Intro" not in body
+
+
+def test_pagination_query_param_respected(seeded_db, csrf_session):
+    """A high page number should still render without crashing."""
+    resp = handle_search({"q": ["example"], "p": ["99"]})
+    assert resp["status"] == 200
+
+
+def test_trusted_sites_fallback_surfaces_when_query_matches_link_label(seeded_db, csrf_session):
+    """Links extracted from indexed pages act as a fallback when direct results
+    are absent or thin; labels are substring-matched case-insensitively."""
+    resp = handle_search({"q": ["advanced"]})
+    body = resp["body"]
+    # The label "advanced rust guide" is on a link extracted from rust-intro.
+    assert "advanced rust guide" in body
+    assert "trusted sites" in body
+
+
+def test_page_count_in_meta_line(seeded_db, csrf_session):
+    resp = handle_search({})
+    assert "4 pages indexed" in resp["body"]
+
+
+def test_csp_and_security_headers_not_in_handler_but_via_dispatch(seeded_db, csrf_session):
+    """Handler itself returns no security headers; dispatch_request wraps them.
+    This test documents the boundary so future refactors don't break assumptions."""
+    resp = handle_search({})
+    assert "headers" not in resp or "Content-Security-Policy" not in resp.get("headers", {})

tests/test_handlers_subs.py

@@ -0,0 +1,112 @@
+"""Tests for subscription handlers.
+
+Subscription add validates the destination hash (32-char hex) locally
+before calling `fetch_remote_sites`; browse uses cached remote_pages when
+available and falls back to a live fetch otherwise.
+"""
+from unittest.mock import patch
+
+import handlers as handlers_module
+from db import get_db, return_db
+from handlers import handle_subscription_add, handle_subscription_browse
+
+
+VALID_HASH = "a" * 32
+
+
+def _subscription_count():
+    db = get_db()
+    try:
+        return db.execute("SELECT count(*) FROM subscriptions").fetchone()[0]
+    finally:
+        return_db(db)
+
+
+def test_rejects_empty_dest_hash(temp_db, csrf_session):
+    resp = handle_subscription_add({"dest_hash": [""]})
+    assert "32-character" in resp["body"]
+    assert _subscription_count() == 0
+
+
+def test_rejects_wrong_length(temp_db, csrf_session):
+    resp = handle_subscription_add({"dest_hash": ["abc123"]})
+    assert "32-character" in resp["body"]
+    assert _subscription_count() == 0
+
+
+def test_rejects_non_hex(temp_db, csrf_session):
+    resp = handle_subscription_add({"dest_hash": ["z" * 32]})
+    assert "hex" in resp["body"].lower()
+    assert _subscription_count() == 0
+
+
+def test_rejects_unreachable_peer(temp_db, csrf_session):
+    with patch.object(handlers_module, "fetch_remote_sites") as fetch:
+        fetch.side_effect = ConnectionError("unreachable")
+        resp = handle_subscription_add({"dest_hash": [VALID_HASH]})
+    assert "Could not reach" in resp["body"]
+    assert _subscription_count() == 0
+
+
+def test_rejects_peer_with_sharing_disabled(temp_db, csrf_session):
+    with patch.object(handlers_module, "fetch_remote_sites") as fetch:
+        fetch.side_effect = PermissionError("sharing disabled")
+        resp = handle_subscription_add({"dest_hash": [VALID_HASH]})
+    assert "sharing disabled" in resp["body"]
+    assert _subscription_count() == 0
+
+
+def test_successful_add_records_subscription(temp_db, csrf_session):
+    with patch.object(handlers_module, "fetch_remote_sites") as fetch:
+        fetch.return_value = {"name": "alice", "sites": []}
+        resp = handle_subscription_add({"dest_hash": [VALID_HASH]})
+    assert "Subscribed to alice" in resp["body"]
+    assert _subscription_count() == 1
+
+
+def test_dest_hash_strips_angle_brackets(temp_db, csrf_session):
+    """Users often paste hashes as `<aaa...>` from RNS log output; strip them."""
+    with patch.object(handlers_module, "fetch_remote_sites") as fetch:
+        fetch.return_value = {"name": "bob", "sites": []}
+        resp = handle_subscription_add({"dest_hash": [f"<{VALID_HASH}>"]})
+    assert _subscription_count() == 1
+
+
+def test_browse_unknown_subscription_is_404(temp_db, csrf_session):
+    resp = handle_subscription_browse(99999)
+    assert resp["status"] == 404
+
+
+def test_browse_marks_already_indexed_urls(seeded_db, csrf_session):
+    # Insert a subscription + some remote pages (one duplicate of local, one new).
+    db = get_db()
+    try:
+        db.execute(
+            "INSERT INTO subscriptions (dest_hash, name) VALUES (?, ?)",
+            (VALID_HASH, "alice"),
+        )
+        sub_id = db.execute("SELECT id FROM subscriptions").fetchone()["id"]
+        db.execute(
+            "INSERT INTO remote_pages (subscription_id, url, title, note, tags) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (sub_id, "https://example.com/rust-intro", "Alice rust pick", "", ""),
+        )
+        db.execute(
+            "INSERT INTO remote_pages (subscription_id, url, title, note, tags) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (sub_id, "https://new.example.com/shiny", "Shiny New Link", "note", "tag1"),
+        )
+        db.commit()
+    finally:
+        return_db(db)
+
+    resp = handle_subscription_browse(sub_id)
+    body = resp["body"]
+    assert resp["status"] == 200
+    assert "already indexed" in body
+    # The duplicate URL should appear in the "already indexed" section.
+    assert "Alice rust pick" in body
+    # The new URL should be in the selectable section.
+    assert "Shiny New Link" in body
+    # Count summary: "2 site(s) available, 1 new"
+    assert "1 new" in body

tests/test_handlers_tags.py

@@ -0,0 +1,101 @@
+"""Tests for tag helpers and the tag browse handler.
+
+Tags are stored via a join table, so orphaned rows in `tags` can accumulate
+if `_cleanup_orphaned_tags` isn't called after deletion/retagging. Tag
+counts shown in the UI rely on this being right.
+"""
+from db import get_db, return_db
+from handlers import (
+    _cleanup_orphaned_tags,
+    _get_page_tags,
+    _set_page_tags,
+    handle_tag_browse,
+    handle_tags,
+)
+
+
+def _page_id(url):
+    db = get_db()
+    try:
+        row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()
+        return row["id"] if row else None
+    finally:
+        return_db(db)
+
+
+def _tag_names():
+    db = get_db()
+    try:
+        return {r["name"] for r in db.execute("SELECT name FROM tags").fetchall()}
+    finally:
+        return_db(db)
+
+
+def test_get_page_tags_returns_sorted_names(seeded_db):
+    pid = _page_id("https://example.com/rust-intro")
+    tags = _get_page_tags(pid)
+    assert tags == sorted(tags)  # alphabetical
+    assert "rust" in tags
+    assert "public" in tags
+
+
+def test_set_page_tags_replaces_existing(seeded_db):
+    pid = _page_id("https://example.com/rust-intro")
+    db = get_db()
+    try:
+        _set_page_tags(pid, "brand, new, tags", db)
+        db.commit()
+    finally:
+        return_db(db)
+    current = _get_page_tags(pid)
+    assert current == ["brand", "new", "tags"]
+
+
+def test_set_page_tags_splits_on_comma_and_lowercases(seeded_db):
+    pid = _page_id("https://example.com/python-tips")
+    db = get_db()
+    try:
+        _set_page_tags(pid, "Foo, BAR, baz", db)
+        db.commit()
+    finally:
+        return_db(db)
+    assert set(_get_page_tags(pid)) == {"foo", "bar", "baz"}
+
+
+def test_cleanup_orphaned_tags_removes_unreferenced(seeded_db):
+    # Clear all tags on one page; previously-unique tags become orphans.
+    pid = _page_id("https://example.com/rust-intro")
+    db = get_db()
+    try:
+        _set_page_tags(pid, "", db)  # empty string = no tags
+        # `rust` was only on the rust-intro page; `public` is also on mesh.
+        _cleanup_orphaned_tags(db)
+        db.commit()
+    finally:
+        return_db(db)
+    names = _tag_names()
+    assert "rust" not in names  # pruned
+    assert "public" in names  # still on mesh
+
+
+def test_handle_tag_browse_filters_by_tag(seeded_db, csrf_session):
+    resp = handle_tag_browse("rust", {})
+    assert resp["status"] == 200
+    body = resp["body"]
+    assert "Rust Intro" in body
+    assert "Python Tips" not in body
+    assert "Why OCaml" not in body
+
+
+def test_handle_tag_browse_unknown_tag_is_graceful(seeded_db, csrf_session):
+    resp = handle_tag_browse("no-such-tag", {})
+    # Should render a valid page with zero results, not error.
+    assert resp["status"] == 200
+
+
+def test_handle_tags_lists_all_tags_with_counts(seeded_db, csrf_session):
+    resp = handle_tags()
+    assert resp["status"] == 200
+    body = resp["body"]
+    for tag in ("rust", "python", "ocaml", "mesh", "public", "private"):
+        assert tag in body

tests/test_link_extraction.py

@@ -0,0 +1,138 @@
+"""Tests for link extraction inside `fetch_page`.
+
+Link extraction powers the "trusted sites" fallback on empty searches and
+feeds the `links` table. Rules: same-domain only, skip binary extensions,
+skip Wikipedia special pages, resolve relatives via urljoin.
+"""
+from unittest.mock import patch
+
+from conftest import patch_dns_ok
+import db as db_module
+
+
+class FakeResponse:
+    def __init__(self, text, status_code=200):
+        self.text = text
+        self.status_code = status_code
+        self.is_redirect = False
+        self.headers = {}
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise Exception(f"status {self.status_code}")
+
+
+def _fetch_with_html(monkeypatch, url, html):
+    """Invoke fetch_page against `url` with `html` as the mocked response body."""
+    patch_dns_ok(monkeypatch)
+    with patch.object(db_module, "requests") as mock_requests:
+        mock_requests.get.return_value = FakeResponse(html)
+        return db_module.fetch_page(url)
+
+
+def test_only_same_domain_links_kept(monkeypatch):
+    html = """
+    <html><body>
+      <a href="https://example.com/a">same</a>
+      <a href="https://other.com/b">cross</a>
+      <a href="https://sub.example.com/c">subdomain</a>
+    </body></html>
+    """
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    urls = [u for u, _label in links]
+    assert "https://example.com/a" in urls
+    assert "https://other.com/b" not in urls
+    assert "https://sub.example.com/c" not in urls
+
+
+def test_binary_extensions_skipped(monkeypatch):
+    html = """
+    <html><body>
+      <a href="/real-page">keep</a>
+      <a href="/image.png">skip</a>
+      <a href="/doc.pdf">skip</a>
+      <a href="/archive.zip">skip</a>
+      <a href="/song.mp3">skip</a>
+      <a href="/styles.css">skip</a>
+    </body></html>
+    """
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    urls = [u for u, _label in links]
+    assert "https://example.com/real-page" in urls
+    for ext in (".png", ".pdf", ".zip", ".mp3", ".css"):
+        assert not any(u.endswith(ext) for u in urls), f"{ext} leaked through"
+
+
+def test_wikipedia_special_pages_skipped(monkeypatch):
+    html = """
+    <html><body>
+      <a href="/wiki/Main_Page">keep</a>
+      <a href="/wiki/Special:Random">skip</a>
+      <a href="/wiki/Talk:Foo">skip</a>
+      <a href="/wiki/User:Jimbo">skip</a>
+      <a href="/wiki/Category:Bar">skip</a>
+    </body></html>
+    """
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    urls = [u for u, _label in links]
+    assert "https://example.com/wiki/Main_Page" in urls
+    for skip in ("Special:Random", "Talk:Foo", "User:Jimbo", "Category:Bar"):
+        assert not any(skip in u for u in urls), f"wiki {skip!r} leaked"
+
+
+def test_relative_urls_resolved(monkeypatch):
+    html = """<html><body><a href="/relative/path">r</a></body></html>"""
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/start", html)
+    urls = [u for u, _label in links]
+    assert "https://example.com/relative/path" in urls
+
+
+def test_fragment_stripped_from_extracted_links(monkeypatch):
+    html = """<html><body><a href="/page#section">r</a></body></html>"""
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    urls = [u for u, _label in links]
+    assert "https://example.com/page" in urls
+    assert not any("#" in u for u in urls)
+
+
+def test_duplicate_links_deduped(monkeypatch):
+    html = """
+    <html><body>
+      <a href="/a">first</a>
+      <a href="/a">second</a>
+      <a href="/a">third</a>
+    </body></html>
+    """
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    urls = [u for u, _label in links]
+    assert urls.count("https://example.com/a") == 1
+
+
+def test_label_truncated_to_200(monkeypatch):
+    long_text = "x" * 500
+    html = f'<html><body><a href="/p">{long_text}</a></body></html>'
+    _, _, links, _ = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    assert len(links) == 1
+    _, label = links[0]
+    assert len(label) <= 200
+
+
+def test_meta_description_extracted(monkeypatch):
+    html = """
+    <html><head>
+      <meta name="description" content="the real description">
+    </head><body><p>body content</p></body></html>
+    """
+    title, body, links, meta = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    assert meta == "the real description"
+
+
+def test_og_description_fallback(monkeypatch):
+    """When there's no <meta name=description>, og:description wins."""
+    html = """
+    <html><head>
+      <meta property="og:description" content="open graph fallback">
+    </head><body><p>body</p></body></html>
+    """
+    _, _, _, meta = _fetch_with_html(monkeypatch, "https://example.com/", html)
+    assert meta == "open graph fallback"

tests/test_pagination.py

@@ -0,0 +1,58 @@
+"""Tests for `_paginate` and `_page_nav`."""
+from handlers import _paginate, _page_nav, PER_PAGE
+
+
+def test_paginate_default_is_one():
+    assert _paginate({}) == 1
+
+
+def test_paginate_reads_query_string():
+    assert _paginate({"p": ["3"]}) == 3
+
+
+def test_paginate_clamps_to_one():
+    assert _paginate({"p": ["0"]}) == 1
+    assert _paginate({"p": ["-5"]}) == 1
+
+
+def test_paginate_handles_bad_input():
+    assert _paginate({"p": ["not-a-number"]}) == 1
+    assert _paginate({"p": []}) == 1
+
+
+def test_paginate_custom_key():
+    assert _paginate({"batch": ["7"]}, key="batch") == 7
+
+
+def test_page_nav_empty_when_single_page():
+    assert _page_nav(1, PER_PAGE, "/?q=foo") == ""
+    assert _page_nav(1, 0, "/?q=foo") == ""
+
+
+def test_page_nav_shows_next_on_first_page():
+    out = _page_nav(1, PER_PAGE * 3, "/?q=foo")
+    assert "next" in out
+    assert "prev" not in out
+    assert "page 1 of 3" in out
+
+
+def test_page_nav_shows_both_in_middle():
+    out = _page_nav(2, PER_PAGE * 3, "/?q=foo")
+    assert "next" in out
+    assert "prev" in out
+
+
+def test_page_nav_shows_prev_on_last_page():
+    out = _page_nav(3, PER_PAGE * 3, "/?q=foo")
+    assert "next" not in out
+    assert "prev" in out
+    assert "page 3 of 3" in out
+
+
+def test_page_nav_handles_query_string_separator():
+    # when base_url already has ?, pagination links must use &
+    out = _page_nav(1, PER_PAGE * 2, "/?q=foo")
+    assert "&p=2" in out
+    # when base_url has no ?, pagination links use ?
+    out = _page_nav(1, PER_PAGE * 2, "/pages")
+    assert "?p=2" in out

tests/test_regressions.py

@@ -0,0 +1,107 @@
+"""Aggregator of regression tests tied to specific bug-fix commits.
+
+Each test here guards against a specific bug that was once shipped. Running
+just this file gives a one-line-per-bug audit:
+
+    pytest tests/test_regressions.py -v
+
+The test bodies are intentionally small; for the exhaustive behavior of each
+module, see the topical test files (test_fts_sanitizer.py, test_url_cleanup.py,
+etc.). This file's job is to make the bug catalog scannable.
+"""
+import socket
+from unittest.mock import patch
+
+import pytest
+
+import app as app_module
+import db as db_module
+import handlers as handlers_module
+from conftest import patch_dns_fail, patch_dns_ok
+from db import clean_url
+from handlers import _sanitize_fts_query, handle_bulk_action
+
+
+def test_6ffd38d_clean_url_preserves_www_when_bare_domain_fails(monkeypatch):
+    """6ffd38d: `clean_url` used to strip `www.` unconditionally; for sites that
+    only serve at `www.`, this produced unreachable clean URLs."""
+    patch_dns_fail(monkeypatch)
+    assert clean_url("https://www.example.com/page") == "https://www.example.com/page"
+
+
+def test_1bc695f_fts_sanitizer_strips_colon():
+    """1bc695f: FTS5 colon is a column filter — must not appear in sanitized output."""
+    assert ":" not in _sanitize_fts_query("title:secret body:exposed")
+
+
+@pytest.mark.parametrize("op", ["AND", "OR", "NOT", "NEAR"])
+def test_1bc695f_fts_sanitizer_drops_operator_words(op):
+    """1bc695f: operator words (AND/OR/NOT/NEAR) would be interpreted as FTS5
+    operators if they landed on the unquoted last token."""
+    out = _sanitize_fts_query(f"foo {op} bar")
+    # operator itself should not appear in the output
+    tokens = out.replace('"', '').split()
+    assert op not in [t.rstrip("*") for t in tokens]
+
+
+def test_1bc695f_gateway_rejects_oversize_body():
+    """1bc695f: 16 MiB body-size cap prevents memory-exhaustion DoS."""
+    from tests.test_gateway_limits import FakeGatewayHandler
+    from gateway import MAX_BODY_SIZE
+    h = FakeGatewayHandler(
+        path="/add", method="POST",
+        headers={"Content-Length": str(MAX_BODY_SIZE + 1)},
+    )
+    h._forward("POST")
+    assert h._captured["error"] and h._captured["error"][0] == 413
+
+
+def test_1bc695f_mesh_rejects_non_whitelisted_paths():
+    """1bc695f: Reticulum callers are limited to GET /api/sites; CSRF cannot
+    authenticate mesh callers."""
+    resp = app_module.rns_request_handler(
+        path="/tinyweb",
+        data={"method": "POST", "path": "/add", "query": {}, "body": {}, "gateway_host": ""},
+        request_id="x", link_id="y", remote_identity=None, requested_at=0,
+    )
+    assert resp["status"] == 403
+
+
+def test_1bc695f_pool_returns_clean_connection(temp_db, monkeypatch):
+    """1bc695f: uncommitted transactions on a pooled connection used to leak
+    into the next consumer."""
+    from db import get_db, return_db
+    db = get_db()
+    db.execute(
+        "INSERT INTO pages (url, title, body) VALUES (?, ?, ?)",
+        ("https://leak.example.com/", "should not persist", "body"),
+    )
+    return_db(db)  # no commit
+    db2 = get_db()
+    try:
+        urls = {r["url"] for r in db2.execute("SELECT url FROM pages").fetchall()}
+    finally:
+        return_db(db2)
+    assert "https://leak.example.com/" not in urls
+
+
+def test_8dffd8c_bulk_delete_requires_confirmation(seeded_db, csrf_session):
+    """8dffd8c: bulk delete without confirmed=1 must render a confirm page
+    instead of deleting — the JS confirm on /pages is a first-line filter only."""
+    from db import get_db, return_db
+    db = get_db()
+    try:
+        pid = db.execute("SELECT id FROM pages LIMIT 1").fetchone()["id"]
+        count_before = db.execute("SELECT count(*) FROM pages").fetchone()[0]
+    finally:
+        return_db(db)
+
+    resp = handle_bulk_action({"ids": [str(pid)], "action": ["delete"]})
+    assert "confirm delete" in resp["body"].lower()
+
+    db = get_db()
+    try:
+        count_after = db.execute("SELECT count(*) FROM pages").fetchone()[0]
+    finally:
+        return_db(db)
+    assert count_before == count_after, "bulk delete ran without confirmation"

tests/test_sharing_logic.py

@@ -0,0 +1,38 @@
+"""Tests for `_page_is_shared`.
+
+This function decides whether a page is exposed over Reticulum to
+subscribers. Getting it wrong means either a privacy leak or silently
+hiding pages the user meant to share — both are worth a regression net.
+"""
+import pytest
+
+from handlers import _page_is_shared
+
+
+@pytest.mark.parametrize("mode", ["exclude_private", "require_public"])
+def test_private_tag_always_excludes(mode):
+    """`private` tag overrides every mode — the most important invariant."""
+    assert _page_is_shared(["private"], mode) is False
+    assert _page_is_shared(["public", "private"], mode) is False
+
+
+def test_exclude_private_defaults_to_shared():
+    assert _page_is_shared([], "exclude_private") is True
+    assert _page_is_shared(["random-tag"], "exclude_private") is True
+
+
+def test_require_public_needs_public_tag():
+    assert _page_is_shared([], "require_public") is False
+    assert _page_is_shared(["rust"], "require_public") is False
+    assert _page_is_shared(["public"], "require_public") is True
+
+
+def test_require_public_still_vetoes_private():
+    # public AND private → private wins.
+    assert _page_is_shared(["public", "private"], "require_public") is False
+
+
+def test_unknown_mode_treated_as_exclude_private():
+    """The default mode is 'exclude_private'; unknown modes fall through to it."""
+    assert _page_is_shared([], "totally-bogus-mode") is True
+    assert _page_is_shared(["private"], "totally-bogus-mode") is False

tests/test_ssrf.py

@@ -0,0 +1,64 @@
+"""Tests for `_validate_url_target` — SSRF prevention.
+
+Any URL the app fetches must resolve to a public IP; private/internal/
+loopback addresses must be rejected so attacker-controlled URLs cannot
+reach internal services via our HTTP client.
+"""
+import socket
+from unittest.mock import patch
+
+import pytest
+
+from db import _validate_url_target
+
+
+def _mock_getaddrinfo(address):
+    """Return a function suitable as a socket.getaddrinfo replacement."""
+    def f(host, port, *args, **kwargs):
+        family = socket.AF_INET6 if ":" in address else socket.AF_INET
+        return [(family, socket.SOCK_STREAM, 0, "", (address, port or 80))]
+    return f
+
+
+@pytest.mark.parametrize("blocked_ip", [
+    "127.0.0.1",
+    "127.1.2.3",
+    "10.0.0.1",
+    "10.255.255.255",
+    "172.16.0.1",
+    "172.31.255.255",
+    "192.168.0.1",
+    "192.168.255.255",
+    "169.254.169.254",
+    "0.0.0.0",
+    "::1",
+    "fc00::1",
+    "fe80::1",
+])
+def test_blocks_private_and_loopback(monkeypatch, blocked_ip):
+    monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo(blocked_ip))
+    with pytest.raises(ValueError, match="blocked"):
+        _validate_url_target("https://evil.example.com/internal")
+
+
+def test_allows_public_ipv4(monkeypatch):
+    monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo("8.8.8.8"))
+    _validate_url_target("https://dns.example.com/")  # does not raise
+
+
+def test_allows_public_ipv6(monkeypatch):
+    monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo("2001:4860:4860::8888"))
+    _validate_url_target("https://v6.example.com/")  # does not raise
+
+
+def test_rejects_unresolvable_hostname(monkeypatch):
+    def boom(*args, **kwargs):
+        raise socket.gaierror("no such host")
+    monkeypatch.setattr(socket, "getaddrinfo", boom)
+    with pytest.raises(ValueError, match="Cannot resolve"):
+        _validate_url_target("https://does-not-exist.example.com/")
+
+
+def test_rejects_missing_hostname():
+    with pytest.raises(ValueError, match="No hostname"):
+        _validate_url_target("http:///path-only")

tests/test_url_cleanup.py

@@ -0,0 +1,101 @@
+"""Tests for `clean_url` — URL normalization and tracking-param stripping.
+
+Clean URLs are the deduplication key in the pages table, so any change to
+this function can silently cause duplicate rows or mask legitimate saves.
+"""
+import pytest
+
+from conftest import patch_dns_ok, patch_dns_fail
+from db import clean_url, TRACKING_PARAMS
+
+
+def test_strips_fragment(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://example.com/page#section") == "https://example.com/page"
+
+
+def test_prefers_https(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("http://example.com/page") == "https://example.com/page"
+
+
+def test_lowercases_hostname(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://EXAMPLE.COM/page") == "https://example.com/page"
+
+
+def test_preserves_path_case(monkeypatch):
+    """Paths are case-sensitive and should not be lowercased."""
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://example.com/Foo/Bar") == "https://example.com/Foo/Bar"
+
+
+def test_strips_default_https_port(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://example.com:443/page") == "https://example.com/page"
+
+
+@pytest.mark.xfail(reason="clean_url upgrades http->https before the port-default check, "
+                          "so port 80 is not stripped. Minor dedup bug — harmless but worth fixing.")
+def test_strips_http_port_80(monkeypatch):
+    """Expected: http://foo:80 → https://foo (both scheme-upgrade and port-strip).
+
+    Currently fails because scheme is upgraded to https *before* the port check,
+    so `scheme == "http" and port == 80` is never true by the time the check runs.
+    """
+    patch_dns_ok(monkeypatch)
+    assert clean_url("http://example.com:80/page") == "https://example.com/page"
+
+
+def test_preserves_non_default_port(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://example.com:8443/page") == "https://example.com:8443/page"
+
+
+def test_strips_trailing_slash(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://example.com/page/") == "https://example.com/page"
+
+
+def test_root_slash_preserved(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://example.com/") == "https://example.com/"
+
+
+@pytest.mark.parametrize("param", sorted(TRACKING_PARAMS))
+def test_tracking_params_stripped(monkeypatch, param):
+    patch_dns_ok(monkeypatch)
+    result = clean_url(f"https://example.com/page?{param}=value&keep=yes")
+    assert param not in result
+    assert "keep=yes" in result
+
+
+def test_strips_www_when_nonwww_resolves(monkeypatch):
+    """Standard case: strip `www.` prefix to canonicalize."""
+    patch_dns_ok(monkeypatch)
+    assert clean_url("https://www.example.com/page") == "https://example.com/page"
+
+
+def test_preserves_www_when_nonwww_does_not_resolve(monkeypatch):
+    """Regression for 6ffd38d.
+
+    Some sites only serve their content at `www.domain.tld`; the bare domain
+    doesn't resolve. Stripping `www.` in that case produced a URL that we could
+    never actually fetch or dedupe against the real one.
+    """
+    patch_dns_fail(monkeypatch)
+    assert clean_url("https://www.example.com/page") == "https://www.example.com/page"
+
+
+def test_query_params_sorted_for_stable_ordering(monkeypatch):
+    """Same URL with different param orderings should produce the same clean URL."""
+    patch_dns_ok(monkeypatch)
+    a = clean_url("https://example.com/page?b=2&a=1")
+    b = clean_url("https://example.com/page?a=1&b=2")
+    assert a == b
+
+
+def test_path_and_query_preserved_through_cleanup(monkeypatch):
+    patch_dns_ok(monkeypatch)
+    result = clean_url("https://example.com/path/to/page?id=42&utm_source=twitter")
+    assert result == "https://example.com/path/to/page?id=42"

themes/kodama.html

@@ -0,0 +1,747 @@
+<!DOCTYPE html>
+<html>
+    <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="referrer" content="no-referrer">
+    <meta http-equiv="x-dns-prefetch-control" content="off">
+    <style>
+
+      * { margin: 0; padding: 0; box-sizing: border-box; }
+
+      body {
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
+        font-size: 16px;
+        line-height: 1.65;
+        color: #c8c8c8;
+        background: #111;
+        cursor: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='20' height='20'%3E%3Ccircle cx='10' cy='10' r='3' fill='none' stroke='%23888' stroke-width='1.5'/%3E%3Ccircle cx='10' cy='10' r='1'  fill='%23aaa'/%3E%3C/svg%3E") 10 10, default;
+      }
+
+      a, button, input[type="submit"], summary, label {
+        cursor: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='20' height='20'%3E%3Ccircle cx='10' cy='10' r='6' fill='none' stroke='%23fff' stroke-width='1' opacity='0.6'/%3E%3Ccircle cx='10' cy='10' r='1.5'  fill='%23fff'/%3E%3C/svg%3E") 10 10, pointer;
+      }
+
+      input, textarea {
+        cursor: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='20' height='20'%3E%3Cline x1='10' y1='3' x2='10' y2='17' stroke='%23aaa' stroke-width='1.5'/%3E%3C/svg%3E") 10 10, text;
+      }
+
+      /* scanline overlay */
+      body::before {
+        content: '';
+        position: fixed;
+        top: 0; left: 0;
+        width: 100%; height: 100%;
+        background: repeating-linear-gradient(
+          0deg,
+          transparent,
+          transparent 2px,
+          rgba(255, 255, 255, 0.008) 2px,
+          rgba(255, 255, 255, 0.008) 4px
+        );
+        pointer-events: none;
+        z-index: 1000;
+      }
+
+      /* vignette */
+      body::after {
+        content: '';
+        position: fixed;
+        top: 0; left: 0;
+        width: 100%; height: 100%;
+        background: radial-gradient(ellipse at center, transparent 60%, rgba(0, 0, 0, 0.4) 100%);
+        pointer-events: none;
+        z-index: 999;
+      }
+
+      /* floating particles canvas */
+      #particles {
+        position: fixed;
+        top: 0; left: 0;
+        width: 100%; height: 100%;
+        pointer-events: none;
+        z-index: 0;
+      }
+
+      /* cursor trail canvas */
+      #trail {
+        position: fixed;
+        top: 0; left: 0;
+        width: 100%; height: 100%;
+        pointer-events: none;
+        z-index: 998;
+      }
+
+
+      /* kodama spirits */
+      #kodama {
+        position: fixed;
+        top: 0; left: 0;
+        width: 100%; height: 100%;
+        pointer-events: none;
+        z-index: 2;
+      }
+
+      .shell {
+        max-width: 660px;
+        margin: 0 auto;
+        padding: 0 1.5rem;
+        position: relative;
+        z-index: 1;
+      }
+
+      /* nav */
+      nav {
+        display: flex;
+        align-items: baseline;
+        justify-content: space-between;
+        padding: 1.5rem 0 1.2rem;
+        border-bottom: 1px solid #232323;
+      }
+
+      nav .site {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.85rem;
+        font-weight: 500;
+        color: #e8e8e8;
+        text-decoration: none;
+        letter-spacing: 0.06em;
+        border-bottom: none;
+        transition: text-shadow 0.3s;
+      }
+
+      nav .site:hover {
+        text-shadow: 0 0 8px rgba(255,255,255,0.3);
+      }
+
+      nav .links { display: flex; gap: 1.2rem; }
+
+      nav .links a {
+        font-size: 0.82rem;
+        color: #606060;
+        text-decoration: none;
+        border-bottom: none;
+        transition: color 0.2s, text-shadow 0.3s;
+      }
+
+      nav .links a:hover {
+        color: #c8c8c8;
+        text-shadow: 0 0 6px rgba(255,255,255,0.15);
+      }
+
+      /* greeting */
+      #greeting {
+        padding: 1.5rem 0 0;
+        font-size: 0.85rem;
+        color: #484848;
+        font-style: italic;
+        opacity: 0;
+        animation: fadeIn 1.5s ease forwards 0.3s;
+      }
+
+      @keyframes fadeIn {
+        to { opacity: 1; }
+      }
+
+      /* content */
+      .content { padding: 1.8rem 0 3rem; }
+
+      /* headings */
+      h1 {
+        font-size: 1.4rem;
+        font-weight: 600;
+        color: #e8e8e8;
+        margin-bottom: 1rem;
+      }
+
+      h1 a { color: #e8e8e8; text-decoration: none; border-bottom: none; }
+      h1 a:hover { color: #999; }
+
+      h2 {
+        font-size: 1.05rem;
+        font-weight: 600;
+        color: #d0d0d0;
+        margin: 1.8rem 0 0.5rem;
+      }
+
+      p { margin: 0.5rem 0; color: #999; }
+
+      a {
+        color: #c8c8c8;
+        text-decoration: none;
+        border-bottom: 1px solid #2e2e2e;
+        transition: all 0.2s;
+      }
+
+      a:hover {
+        color: #fff;
+        border-bottom-color: #555;
+      }
+
+      em { color: #777; }
+
+      /* inputs */
+      input[type="text"],
+      input[type="url"],
+      input[name="q"],
+      input[name="url"],
+      input[name="note"],
+      input[name="tags"],
+      input[name="site_name"],
+      input[name="dest_hash"] {
+        background: #181818;
+        border: 1px solid #2a2a2a;
+        border-radius: 4px;
+        padding: 0.6rem 0.85rem;
+        color: #d0d0d0;
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
+        font-size: 0.95rem;
+        transition: border-color 0.2s, box-shadow 0.3s;
+      }
+
+      input:focus, textarea:focus {
+        outline: none;
+        border-color: #454545;
+        box-shadow: 0 0 12px rgba(255,255,255,0.03);
+      }
+
+      button, input[type="submit"] {
+        background: #1c1c1c;
+        border: 1px solid #303030;
+        border-radius: 4px;
+        padding: 0.6rem 1.1rem;
+        color: #999;
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
+        font-size: 0.88rem;
+        transition: all 0.2s;
+      }
+
+      button:hover, input[type="submit"]:hover {
+        background: #242424;
+        color: #d0d0d0;
+        border-color: #454545;
+        box-shadow: 0 0 10px rgba(255,255,255,0.04);
+      }
+
+      /* search results */
+      .result {
+        padding: 1rem 0;
+        border-bottom: 1px solid #1c1c1c;
+        transition: background 0.2s;
+      }
+
+      .result:hover {
+        background: rgba(255,255,255,0.01);
+      }
+
+      .result:last-child { border-bottom: none; }
+
+      .result > a:first-child {
+        font-size: 1.02rem;
+        font-weight: 500;
+        color: #ddd;
+        border-bottom: none;
+      }
+
+      .result > a:first-child:hover { color: #fff; }
+
+      .note {
+        margin-top: 0.3rem;
+        font-size: 0.9rem;
+        color: #606060;
+      }
+
+      .tags { margin-top: 0.3rem; }
+
+      .tag, .tags a {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.7rem;
+        color: #555;
+        border: 1px solid #252525;
+        border-radius: 3px;
+        padding: 0.1rem 0.35rem;
+        margin-right: 0.25rem;
+      }
+
+      .tag:hover, .tags a:hover {
+        color: #999;
+        border-color: #404040;
+      }
+
+      /* trusted / remote results */
+      details {
+        margin: 1rem 0;
+        border: 1px solid #1e1e1e;
+        border-radius: 4px;
+        padding: 0.7rem 0.9rem;
+        background: #151515;
+      }
+
+      summary {
+        font-size: 0.85rem;
+        color: #606060;
+        font-weight: 500;
+      }
+
+      summary:hover { color: #999; }
+
+      details ul { margin-top: 0.5rem; padding-left: 1.2rem; }
+      details li { margin: 0.35rem 0; font-size: 0.9rem; }
+
+      /* lists */
+      ul, ol { padding-left: 1.2rem; margin: 0.5rem 0; }
+
+      li { margin: 0.45rem 0; color: #999; }
+      li a { border-bottom: none; }
+      li a:hover { border-bottom: 1px solid #444; }
+
+      /* code */
+      pre {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.8rem;
+        background: #151515;
+        border: 1px solid #232323;
+        border-radius: 4px;
+        padding: 0.9rem;
+        overflow-x: auto;
+        color: #808080;
+        margin: 0.8rem 0;
+      }
+
+      code {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.82rem;
+        background: #1a1a1a;
+        border-radius: 3px;
+        padding: 0.1rem 0.35rem;
+        color: #999;
+      }
+
+      /* textarea */
+      textarea {
+        background: #151515;
+        border: 1px solid #2a2a2a;
+        border-radius: 4px;
+        padding: 0.7rem 0.9rem;
+        color: #c8c8c8;
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.8rem;
+        line-height: 1.6;
+        resize: vertical;
+        width: 100%;
+      }
+
+      /* tables */
+      table { width: 100%; border-collapse: collapse; margin: 1rem 0; }
+
+      th {
+        text-align: left;
+        font-size: 0.72rem;
+        font-weight: 500;
+        color: #505050;
+        text-transform: uppercase;
+        letter-spacing: 0.08em;
+        padding: 0.5rem 0.7rem;
+        border-bottom: 1px solid #232323;
+      }
+
+      td {
+        padding: 0.5rem 0.7rem;
+        border-bottom: 1px solid #191919;
+        font-size: 0.9rem;
+      }
+
+      /* misc */
+      label { color: #999; }
+      input[type="checkbox"] { accent-color: #555; }
+
+      hr { border: none; border-top: 1px solid #1e1e1e; margin: 1rem 0; }
+
+      small {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.7rem;
+        color: #484848;
+      }
+
+      /* footer */
+      footer {
+        border-top: 1px solid #1c1c1c;
+        padding: 1.5rem 0 2rem;
+        text-align: center;
+        color: #333;
+        font-size: 0.8rem;
+      }
+
+      footer .clock {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 0.72rem;
+        color: #282828;
+        margin-top: 0.25rem;
+      }
+
+      ::selection { background: #333; color: #fff; }
+
+      ::-webkit-scrollbar { width: 5px; }
+      ::-webkit-scrollbar-track { background: transparent; }
+      ::-webkit-scrollbar-thumb { background: #222; border-radius: 3px; }
+
+      @media (max-width: 600px) {
+        nav { flex-direction: column; gap: 0.5rem; }
+        nav .links { gap: 0.8rem; flex-wrap: wrap; }
+        h1 { font-size: 1.2rem; }
+      }
+    </style>
+    </head>
+    <body>
+    <canvas id="particles"></canvas>
+    <canvas id="trail"></canvas>
+    <canvas id="kodama"></canvas>
+    <div class="shell">
+      <nav>
+        <a class="site" href="/">tinyweb</a>
+        <div class="links">
+          <a href="/pages">browse</a>
+          <a href="/tags">tags</a>
+          <a href="/subscriptions">network</a>
+          <a href="/style">customize</a>
+          <a href="/about">about</a>
+        </div>
+      </nav>
+      <div id="greeting"></div>
+      <div class="content">
+        {{content}}
+      </div>
+      <footer>
+        <div>curated by hand · shared over mesh</div>
+        <div class="clock" id="clock"></div>
+      </footer>
+    </div>
+    <script>
+    (function() {
+      // greeting
+      var h = new Date().getHours();
+      var g = h < 5  ? "still up? the quiet hours are good for finding things." :
+              h < 12 ? "morning. what are you looking for?" :
+              h < 17 ? "afternoon. the index is ready." :
+              h < 21 ? "evening. settle in." :
+                       "late night. good browsing ahead.";
+      document.getElementById('greeting').textContent = g;
+
+      // clock
+      function tick() {
+        var d = new Date();
+        var el = document.getElementById('clock');
+        if (el) el.textContent = d.toLocaleTimeString([], {hour:'2-digit', minute:'2-digit'});
+      }
+      tick();
+      setInterval(tick, 30000);
+
+      // floating dust particles
+      var pc = document.getElementById('particles');
+      var pctx = pc.getContext('2d');
+      var dots = [];
+
+      function resizeParticles() {
+        pc.width = window.innerWidth;
+        pc.height = window.innerHeight;
+      }
+      resizeParticles();
+      window.addEventListener('resize', resizeParticles);
+
+      for (var i = 0; i < 40; i++) {
+        dots.push({
+          x: Math.random() * pc.width,
+          y: Math.random() * pc.height,
+          vy: -(Math.random() * 0.15 + 0.05),
+          vx: (Math.random() - 0.5) * 0.1,
+          r: Math.random() * 1.2 + 0.3,
+          o: Math.random() * 0.25 + 0.05,
+          drift: Math.random() * Math.PI * 2
+        });
+      }
+
+      function drawParticles() {
+        pctx.clearRect(0, 0, pc.width, pc.height);
+        var t = Date.now() * 0.001;
+        for (var i = 0; i < dots.length; i++) {
+          var d = dots[i];
+          d.x += d.vx + Math.sin(t + d.drift) * 0.05;
+          d.y += d.vy;
+          if (d.y < -5) { d.y = pc.height + 5; d.x = Math.random() * pc.width; }
+          if (d.x < -5) d.x = pc.width + 5;
+          if (d.x > pc.width + 5) d.x = -5;
+          var flicker = d.o * (0.6 + 0.4 * Math.sin(t * 1.5 + d.drift));
+          pctx.beginPath();
+          pctx.arc(d.x, d.y, d.r, 0, Math.PI * 2);
+          pctx.fillStyle = 'rgba(200, 200, 200, ' + flicker + ')';
+          pctx.fill();
+        }
+        requestAnimationFrame(drawParticles);
+      }
+      drawParticles();
+
+      // cursor trail
+      var tc = document.getElementById('trail');
+      var tctx = tc.getContext('2d');
+      var points = [];
+      var mx = 0, my = 0;
+
+      function resizeTrail() {
+        tc.width = window.innerWidth;
+        tc.height = window.innerHeight;
+      }
+      resizeTrail();
+      window.addEventListener('resize', resizeTrail);
+
+      document.addEventListener('mousemove', function(e) {
+        mx = e.clientX;
+        my = e.clientY;
+        points.push({ x: mx, y: my, t: Date.now() });
+        if (points.length > 30) points.shift();
+      });
+
+      function drawTrail() {
+        tctx.clearRect(0, 0, tc.width, tc.height);
+        var now = Date.now();
+        // fade out points older than 400ms
+        while (points.length && now - points[0].t > 400) points.shift();
+        if (points.length > 1) {
+          for (var i = 1; i < points.length; i++) {
+            var age = (now - points[i].t) / 400;
+            var alpha = (1 - age) * 0.25;
+            var width = (1 - age) * 2;
+            tctx.beginPath();
+            tctx.moveTo(points[i-1].x, points[i-1].y);
+            tctx.lineTo(points[i].x, points[i].y);
+            tctx.strokeStyle = 'rgba(200, 200, 200, ' + alpha + ')';
+            tctx.lineWidth = width;
+            tctx.lineCap = 'round';
+            tctx.stroke();
+          }
+        }
+        requestAnimationFrame(drawTrail);
+      }
+      drawTrail();
+
+      // kodama (tree spirits)
+      var kc = document.getElementById('kodama');
+      var kctx = kc.getContext('2d');
+      var spirits = [];
+      var numSpirits = 8;
+
+      function resizeKodama() {
+        kc.width = window.innerWidth;
+        kc.height = window.innerHeight;
+      }
+      resizeKodama();
+      window.addEventListener('resize', resizeKodama);
+
+      for (var i = 0; i < numSpirits; i++) {
+        // each spirit gets unique proportions
+        var headR = 0.38 + Math.random() * 0.12; // head radius ratio (bigger = bigger head)
+        var bodyH = 0.25 + Math.random() * 0.2;  // body height ratio
+        var bodyW = 0.3 + Math.random() * 0.15;   // body width ratio
+        var eyeSpread = 0.18 + Math.random() * 0.1; // how far apart eyes are
+        var eyeSize = 0.055 + Math.random() * 0.03; // eye dot size
+        var eyeY = -0.08 + Math.random() * 0.08;    // eye vertical position
+        var hasMouth = Math.random() > 0.3;          // 70% have visible mouth
+        var mouthSize = 0.03 + Math.random() * 0.03;
+        var mouthY = 0.15 + Math.random() * 0.1;
+        var hasArms = Math.random() > 0.4;           // 60% have little arm bumps
+        var glowSize = 1.4 + Math.random() * 0.8;   // glow radius multiplier
+        var glowAlpha = 0.08 + Math.random() * 0.12; // glow brightness
+        var tint = Math.floor(Math.random() * 15);   // slight warm/cool variation
+
+        spirits.push({
+          x: Math.random() * 0.8 + 0.1,
+          baseY: 0.75 + Math.random() * 0.18,
+          size: 10 + Math.random() * 12,
+          phase: Math.random() * Math.PI * 2,
+          tiltSpeed: 1.2 + Math.random() * 2,
+          bobSpeed: 0.6 + Math.random() * 0.8,
+          opacity: 0,
+          targetOpacity: 0.4 + Math.random() * 0.45,
+          fadeSpeed: 0.002 + Math.random() * 0.004,
+          appearing: true,
+          timer: Math.random() * 600,
+          lifespan: 500 + Math.random() * 600,
+          rattleTime: 0,
+          rattling: false,
+          // unique shape params
+          headR: headR,
+          bodyH: bodyH,
+          bodyW: bodyW,
+          eyeSpread: eyeSpread,
+          eyeSize: eyeSize,
+          eyeY: eyeY,
+          hasMouth: hasMouth,
+          mouthSize: mouthSize,
+          mouthY: mouthY,
+          hasArms: hasArms,
+          glowSize: glowSize,
+          glowAlpha: glowAlpha,
+          tint: tint,
+          // 8 offsets that warp the head into a unique rock-like blob
+          hw: [
+            (Math.random()-0.5)*0.25, (Math.random()-0.5)*0.2,
+            (Math.random()-0.5)*0.2,  (Math.random()-0.5)*0.25,
+            (Math.random()-0.5)*0.25, (Math.random()-0.5)*0.2,
+            (Math.random()-0.5)*0.2,  (Math.random()-0.5)*0.25
+          ],
+          headTall: 0.8 + Math.random() * 0.5 // overall tall vs wide
+        });
+      }
+
+      function drawKodamaSpirit(x, y, size, tilt, opacity, sp) {
+        kctx.save();
+        kctx.translate(x, y);
+        kctx.globalAlpha = opacity;
+
+        var r = size * sp.headR; // head radius
+
+        // outer glow aura
+        var grd = kctx.createRadialGradient(0, -size * 0.1, r * 0.3, 0, -size * 0.1, r * sp.glowSize);
+        grd.addColorStop(0, 'rgba(255, 255, 250, ' + sp.glowAlpha + ')');
+        grd.addColorStop(0.5, 'rgba(255, 255, 250, ' + (sp.glowAlpha * 0.3) + ')');
+        grd.addColorStop(1, 'rgba(255, 255, 250, 0)');
+        kctx.fillStyle = grd;
+        kctx.beginPath();
+        kctx.arc(0, -size * 0.1, r * sp.glowSize, 0, Math.PI * 2);
+        kctx.fill();
+
+        // body - stubby rounded shape
+        var bw = size * sp.bodyW;
+        var bh = size * sp.bodyH;
+        var by = size * 0.15;
+        kctx.fillStyle = 'rgb(' + (238 + sp.tint) + ',' + (237 + sp.tint) + ',' + (230 + sp.tint) + ')';
+        kctx.beginPath();
+        kctx.ellipse(0, by + bh * 0.4, bw * 0.5, bh * 0.55, 0, 0, Math.PI * 2);
+        kctx.fill();
+
+        // arms - tiny bumps on sides
+        if (sp.hasArms) {
+          kctx.beginPath();
+          kctx.ellipse(-bw * 0.5 - size * 0.04, by + bh * 0.1, size * 0.05, size * 0.04, -0.3, 0, Math.PI * 2);
+          kctx.fill();
+          kctx.beginPath();
+          kctx.ellipse(bw * 0.5 + size * 0.04, by + bh * 0.1, size * 0.05, size * 0.04, 0.3, 0, Math.PI * 2);
+          kctx.fill();
+        }
+
+        // head (tilts)
+        kctx.save();
+        kctx.rotate(tilt);
+
+        // head - unique rock-like blob shape per spirit
+        var hx = 0, hy = -size * 0.15;
+        var rx = r, ry = r * sp.headTall;
+        var w = sp.hw;
+        kctx.fillStyle = 'rgb(' + (243 + sp.tint) + ',' + (242 + sp.tint) + ',' + (237 + sp.tint) + ')';
+        kctx.beginPath();
+        // top
+        kctx.moveTo(hx + r * w[0], hy - ry);
+        // top-right
+        kctx.bezierCurveTo(
+          hx + rx * (0.55 + w[0]), hy - ry * (0.9 + w[1]),
+          hx + rx * (1.0 + w[1]),  hy - ry * (0.4 + w[0]),
+          hx + rx * (1.0 + w[2]),  hy + ry * w[2]);
+        // bottom-right
+        kctx.bezierCurveTo(
+          hx + rx * (1.0 + w[3]), hy + ry * (0.5 + w[2]),
+          hx + rx * (0.5 + w[3]), hy + ry * (1.0 + w[3]),
+          hx + r * w[4],          hy + ry * (0.95 + w[4] * 0.3));
+        // bottom-left
+        kctx.bezierCurveTo(
+          hx - rx * (0.5 + w[5]), hy + ry * (1.0 + w[5]),
+          hx - rx * (1.0 + w[5]), hy + ry * (0.5 + w[4]),
+          hx - rx * (1.0 + w[6]), hy + ry * w[6]);
+        // top-left
+        kctx.bezierCurveTo(
+          hx - rx * (1.0 + w[7]), hy - ry * (0.4 + w[6]),
+          hx - rx * (0.55 + w[7]),hy - ry * (0.9 + w[7]),
+          hx + r * w[0],          hy - ry);
+        kctx.fill();
+
+        // subtle inner highlight
+        kctx.fillStyle = 'rgba(255, 255, 252, 0.25)';
+        kctx.beginPath();
+        kctx.arc(-rx * 0.12, hy - ry * 0.1, r * 0.45, 0, Math.PI * 2);
+        kctx.fill();
+
+        // eyes - small round dark dots
+        kctx.fillStyle = 'rgba(15, 15, 15, 0.9)';
+        var ey = -size * 0.15 + size * sp.eyeY;
+        var es = size * sp.eyeSize;
+        kctx.beginPath();
+        kctx.arc(-size * sp.eyeSpread, ey, es, 0, Math.PI * 2);
+        kctx.fill();
+        kctx.beginPath();
+        kctx.arc(size * sp.eyeSpread, ey, es, 0, Math.PI * 2);
+        kctx.fill();
+
+        // mouth - tiny dot, not all have one
+        if (sp.hasMouth) {
+          kctx.fillStyle = 'rgba(15, 15, 15, 0.7)';
+          kctx.beginPath();
+          kctx.arc(0, ey + size * sp.mouthY, size * sp.mouthSize, 0, Math.PI * 2);
+          kctx.fill();
+        }
+
+        kctx.restore(); // head tilt
+        kctx.restore(); // position
+      }
+
+      function drawKodama() {
+        kctx.clearRect(0, 0, kc.width, kc.height);
+        var t = Date.now() * 0.001;
+
+        for (var i = 0; i < spirits.length; i++) {
+          var s = spirits[i];
+          s.timer++;
+
+          if (s.appearing) {
+            s.opacity += s.fadeSpeed;
+            if (s.opacity >= s.targetOpacity) s.opacity = s.targetOpacity;
+            if (s.timer > s.lifespan) s.appearing = false;
+          } else {
+            s.opacity -= s.fadeSpeed;
+            if (s.opacity <= 0) {
+              s.opacity = 0;
+              s.x = Math.random() * 0.8 + 0.1;
+              s.baseY = 0.75 + Math.random() * 0.18;
+              s.targetOpacity = 0.4 + Math.random() * 0.45;
+              s.timer = 0;
+              s.lifespan = 500 + Math.random() * 600;
+              s.appearing = true;
+              s.rattleTime = 0;
+            }
+          }
+
+          if (s.opacity <= 0) continue;
+
+          if (!s.rattling && Math.random() < 0.004) {
+            s.rattling = true;
+            s.rattleTime = 0;
+          }
+
+          var tilt = Math.sin(t * s.tiltSpeed + s.phase) * 0.1;
+          if (s.rattling) {
+            s.rattleTime++;
+            tilt = Math.sin(s.rattleTime * 0.9) * 0.35 * Math.max(0, 1 - s.rattleTime / 25);
+            if (s.rattleTime > 25) s.rattling = false;
+          }
+
+          var bobY = Math.sin(t * s.bobSpeed + s.phase) * 2.5;
+          var px = s.x * kc.width;
+          var py = s.baseY * kc.height + bobY;
+
+          drawKodamaSpirit(px, py, s.size, tilt, s.opacity, s);
+        }
+        requestAnimationFrame(drawKodama);
+      }
+      drawKodama();
+
+    })();
+    </script>
+    </body>
+    </html>

themes/tinyweb-site.html

@@ -0,0 +1,292 @@
+<!DOCTYPE html>
+<html>
+    <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="referrer" content="no-referrer">
+    <meta http-equiv="x-dns-prefetch-control" content="off">
+    <style>
+      * { margin: 0; padding: 0; box-sizing: border-box; }
+      html, body { min-height: 100vh; }
+      body {
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
+        font-size: 18px;
+        line-height: 1.6;
+        color: #444;
+        background: #fff;
+        padding: 60px 40px;
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+      }
+      a { color: #444; text-decoration: none; cursor: pointer; }
+      a:hover { color: #222; }
+      input, textarea {
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
+      }
+      input[type="text"],
+      input[type="url"],
+      input[type="search"],
+      input:not([type]),
+      textarea, select {
+        border: 1px solid #ccc;
+        padding: 8px 12px;
+        font-size: 15px;
+        border-radius: 0;
+        background: #fff;
+        color: #444;
+      }
+      input:focus, textarea:focus {
+        outline: none;
+        border-color: #999;
+      }
+      button, input[type="submit"] {
+        border: 1px solid #ccc;
+        padding: 8px 18px;
+        font-size: 13px;
+        text-transform: uppercase;
+        background: #fff;
+        color: #444;
+        cursor: pointer;
+        border-radius: 0;
+        transition: background 0.2s;
+      }
+      button:hover, input[type="submit"]:hover {
+        background: #f5f5f5;
+      }
+      .shell {
+        width: 100%;
+        max-width: 650px;
+      }
+      nav {
+        display: flex;
+        align-items: baseline;
+        justify-content: space-between;
+        margin-bottom: 30px;
+        flex-wrap: wrap;
+        gap: 12px;
+      }
+      nav .site {
+        font-size: 28px;
+        font-weight: bold;
+        color: #222;
+        text-decoration: none;
+        border-bottom: none;
+      }
+      nav .site:hover { color: #222; }
+      nav .links { display: flex; gap: 8px; flex-wrap: wrap; }
+      nav .links a {
+        font-size: 13px;
+        text-transform: uppercase;
+        padding: 6px 14px;
+        border: 1px solid #ccc;
+        background: #fff;
+        color: #444;
+        text-decoration: none;
+        transition: background 0.2s;
+      }
+      nav .links a:hover {
+        background: #f5f5f5;
+        color: #444;
+      }
+      .content { width: 100%; animation: fadeIn 0.3s ease; }
+      @keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
+      h1 {
+        font-size: 32px;
+        font-weight: bold;
+        color: #222;
+        margin-bottom: 20px;
+        line-height: 1.2;
+      }
+      h1 a { color: #222; text-decoration: none; }
+      h2 {
+        font-size: 22px;
+        font-weight: bold;
+        color: #222;
+        margin: 24px 0 12px;
+      }
+      p { margin: 12px 0; color: #444; }
+      a { color: #444; text-decoration: none; border-bottom: 1px solid #ddd; }
+      a:hover { color: #222; border-bottom-color: #999; }
+      em { color: #666; }
+      .result {
+        padding: 20px 0;
+        border-bottom: 1px solid #eee;
+      }
+      .result:last-child { border-bottom: none; }
+      .result > a:first-child {
+        font-size: 18px;
+        color: #222;
+        border-bottom: none;
+      }
+      .result > a:first-child:hover { color: #222; }
+      .note { margin-top: 4px; font-size: 15px; color: #666; }
+      .meta {
+        font-size: 13px;
+        color: #999;
+      }
+      .pagination {
+        font-size: 13px;
+        color: #999;
+        margin: 24px 0;
+        text-align: center;
+      }
+      .pagination a {
+        color: #444;
+        border-bottom: 1px solid #ccc;
+      }
+      .pagination a:hover { color: #222; }
+      .success {
+        color: #444;
+        background: #f5f5f5;
+        border: 1px solid #ddd;
+        padding: 10px 16px;
+        font-size: 15px;
+      }
+      .tags { margin-top: 4px; }
+      .tag, .tags a {
+        font-size: 12px;
+        color: #999;
+        border: 1px solid #ddd;
+        padding: 2px 8px;
+        margin-right: 4px;
+        text-decoration: none;
+      }
+      .tag:hover, .tags a:hover {
+        color: #444;
+        border-color: #ccc;
+      }
+      details {
+        margin: 16px 0;
+        border: 1px solid #eee;
+        padding: 12px 16px;
+        background: #fafafa;
+      }
+      summary { font-size: 15px; color: #666; cursor: pointer; }
+      summary:hover { color: #444; }
+      details ul { margin-top: 8px; padding-left: 20px; }
+      details li { margin: 6px 0; font-size: 15px; }
+      ul, ol { padding-left: 20px; margin: 8px 0; }
+      li { margin: 6px 0; color: #444; }
+      li a { border-bottom: none; }
+      pre {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 14px;
+        background: #f5f5f5;
+        border: 1px solid #eee;
+        padding: 16px;
+        overflow-x: auto;
+        color: #444;
+        margin: 12px 0;
+      }
+      code {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 14px;
+        background: #f5f5f5;
+        padding: 2px 6px;
+        color: #444;
+      }
+      textarea {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 14px;
+        line-height: 1.6;
+        resize: vertical;
+        width: 100%;
+        border: 1px solid #ccc;
+        padding: 10px 12px;
+        background: #fff;
+        color: #444;
+      }
+      table { width: 100%; border-collapse: collapse; margin: 16px 0; }
+      th {
+        text-align: left;
+        font-size: 12px;
+        font-weight: 600;
+        color: #999;
+        text-transform: uppercase;
+        letter-spacing: 0.05em;
+        padding: 8px 12px;
+        border-bottom: 1px solid #eee;
+      }
+      td {
+        padding: 8px 12px;
+        border-bottom: 1px solid #f5f5f5;
+        font-size: 15px;
+      }
+      label { color: #666; }
+      input[type="checkbox"] { accent-color: #444; }
+      .forum-form input, .forum-form button, .forum-toolbar input { border-radius: 0; }
+      .forum-actions a, a.forum-action, a.forum-action-inline {
+        border: 1px solid #ccc; padding: 6px 14px; text-transform: uppercase; font-size: 13px;
+      }
+      .forum-actions a:hover, a.forum-action:hover, a.forum-action-inline:hover {
+        background: #f5f5f5;
+      }
+      a.forum-action-inline { text-transform: none; font-size: 13px; padding: 2px 6px; border: none; }
+      hr { border: none; border-top: 1px solid #eee; margin: 16px 0; }
+
+      small {
+        font-size: 13px;
+        color: #999;
+      }
+      footer {
+        width: 100%;
+        margin-top: 40px;
+        padding-top: 16px;
+        border-top: 1px solid #eee;
+        text-align: center;
+        color: #999;
+        font-size: 13px;
+      }
+      footer .clock {
+        font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace;
+        font-size: 12px;
+        color: #ccc;
+        margin-top: 4px;
+      }
+      ::selection { background: #222; color: #fff; }
+      @media (max-width: 600px) {
+        body { padding: 30px 20px; font-size: 16px; }
+        h1 { font-size: 26px; }
+        nav { flex-direction: column; align-items: flex-start; }
+        nav .links { gap: 6px; }
+        nav .links a { padding: 5px 10px; font-size: 12px; }
+      }
+    </style>
+    </head>
+    <body>
+    <div class="shell">
+      <nav>
+        <a class="site" href="/">{{site_name}}</a>
+        <div class="links">
+          <a href="/pages">browse</a>
+          <a href="/tags">tags</a>
+          <a href="/subscriptions">network</a>
+          {{forum_link}}
+          <a href="/style">customize</a>
+          <a href="/about">about</a>
+        </div>
+      </nav>
+      <div class="content">
+        {{content}}
+      </div>
+      <footer>
+        <div>curated by hand &middot; shared over mesh</div>
+        <div class="clock" id="clock"></div>
+      </footer>
+    </div>
+    <script>
+    (function() {
+      var d = document.getElementById('clock');
+      if (d) {
+        function tick() {
+          var n = new Date();
+          d.textContent = n.toLocaleString();
+        }
+        tick();
+        setInterval(tick, 1000);
+      }
+    })();
+    </script>
+    </body>
+</html>