diff --git a/.github/workflows/production.yaml b/.github/workflows/production.yaml index 7e37313c88..e9d4742b0c 100644 --- a/.github/workflows/production.yaml +++ b/.github/workflows/production.yaml @@ -54,15 +54,41 @@ jobs: firebaseServiceAccount: '${{ secrets.FIREBASE_CI_SERVICE_ACCOUNT }}' channelId: live - # Deploy to Arweave + # Source maps are for Firebase, which has deployed above. On Arweave they + # would only add 11 MB to pay for. + - name: Leave source maps out of the Arweave deploy + run: find build/web -name '*.map' -delete + + # Deploy to Arweave, gzipped: each file is stored compressed with a + # `Content-Encoding: gzip` tag, which gateways send with the bytes, so a + # first visit downloads about 20 MB instead of 77 MB. Library pages load + # this bundle by its manifest ID (docs/LIBRARY_PAGES.md). Pinned to the + # v2.1.2 commit, the first to install the newest Node 22 rather than + # the runner's cached one (CVE-2026-111497, the brotli decoder); the + # action installs @ar.io/deploy@^2 itself. - name: Deploy to Arweave - uses: ar-io/ar-io-deploy@v1 + id: arweave + uses: ar-io/ar-io-deploy@a7290af1906eeea3acc4f4a96d2a53cd7eb625bd # v2.1.2 with: deploy-key: ${{ secrets.DEPLOY_KEY }} arns-key: ${{ secrets.ARNS_KEY }} arns-name: ${{ secrets.ARNS_NAME }} undername: ${{ secrets.DEPLOY_ANT_UNDERNAME }} deploy-folder: ./build/web + compress: gzip + + - name: Record the Arweave manifest + env: + MANIFEST_ID: ${{ steps.arweave.outputs.tx-id }} + run: | + echo "Production on Arweave ($GITHUB_SHA): [$MANIFEST_ID](https://turbo-gateway.com/$MANIFEST_ID/)" >> "$GITHUB_STEP_SUMMARY" + + # A miss here does not undo the deploy, so it is reported, not fatal. + - name: Warm the gateways + continue-on-error: true + env: + MANIFEST_ID: ${{ steps.arweave.outputs.tx-id }} + run: bash tool/warm_arweave_manifest.sh "$MANIFEST_ID" build-ios: diff --git a/.github/workflows/staging-arweave.yaml b/.github/workflows/staging-arweave.yaml new file mode 100644 index 0000000000..cf9767ab85 --- /dev/null +++ b/.github/workflows/staging-arweave.yaml @@ -0,0 +1,85 @@ +name: 'Staging build to Arweave' + +# Uploads a staging build of the chosen branch to Arweave, gzipped, for +# trying library mode on Arweave before a release (docs/LIBRARY_PAGES.md). +# Manual only: every run is paid for by the deploy wallet, and pushes to dev +# deploy staging.ardrive.io through staging.yaml, not here. There is no ArNS +# name; the manifest ID is in the job summary. +on: + workflow_dispatch: + +jobs: + upload: + runs-on: ubuntu-latest + permissions: + contents: read + concurrency: + group: staging-arweave + cancel-in-progress: false + # Every action is pinned by commit: this job holds DEPLOY_KEY, and any + # step before the upload can change what it uploads. + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + # Set Flutter version from fvm + - uses: kuhnroyal/flutter-fvm-config-action@6ffa30473b346f7d7c63cf9e03e6a886f940a72b # v1.2.0 + + # Install Flutter + - uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2.23.0 + with: + channel: ${{ env.FLUTTER_CHANNEL }} + flutter-version: ${{ env.FLUTTER_VERSION }} + + - name: Install script_runner + run: flutter pub global activate script_runner + + # The staging build, without source maps: on Arweave they would only + # add 11 MB to pay for. + - name: Build app + run: | + scr setup + flutter config --enable-web + flutter build web --dart-define=environment=staging --release --pwa-strategy=none --no-web-resources-cdn --dart2js-optimization=O1 + + # An exact, patched Node: '22' alone can resolve to an older 22.x the + # runner image already has, and the CLI decodes brotli responses + # (CVE-2026-111497, fixed in 22.23.4 and 24.22.1). + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '24.22.1' + + # The action has no upload-only mode (it always updates an ArNS name), + # so this runs the CLI at an exact version. --incremental reuses files + # this wallet already uploaded, so a run pays mostly for main.dart.js. + - name: Upload to Arweave + id: upload + env: + DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} + NO_COLOR: '1' + run: | + # Without a `shell:`, steps run without pipefail, and `tee` would + # hide a failed upload. + set -o pipefail + # The CLI takes an Arweave JWK base64-encoded, as the action does. + if [[ "$DEPLOY_KEY" =~ ^\{.*\}$ ]]; then + DEPLOY_KEY=$(printf '%s' "$DEPLOY_KEY" | base64 -w0) + fi + export DEPLOY_KEY + npx --yes @ar.io/deploy@2.1.2 upload --deploy-folder build/web \ + --compress gzip --incremental | tee upload.log + id=$(sed -n 's/^Tx ID: \([A-Za-z0-9_-]\{43\}\)$/\1/p' upload.log | head -1) + if [ -z "$id" ]; then + echo "::error::The upload printed no Tx ID." + exit 1 + fi + echo "tx-id=$id" >> "$GITHUB_OUTPUT" + echo "Staging on Arweave ($GITHUB_REF_NAME, $GITHUB_SHA): [$id](https://turbo-gateway.com/$id/)" >> "$GITHUB_STEP_SUMMARY" + + # A miss here does not undo the upload, so it is reported, not fatal. + - name: Warm the gateways + continue-on-error: true + env: + MANIFEST_ID: ${{ steps.upload.outputs.tx-id }} + run: bash tool/warm_arweave_manifest.sh "$MANIFEST_ID" diff --git a/CLAUDE.md b/CLAUDE.md index 0cac94ce19..308bedfa01 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -324,6 +324,17 @@ Users see "deployment" for an Arweave manifest the app creates or updates (New d breaks the shared history and makes the next release PR conflict. - `dev`'s `pubspec.yaml` version is not bumped after releases, so staging reports an older version than production. +- Production also puts the build on Arweave **gzipped** (`--compress gzip`: each file is + stored compressed with a `Content-Encoding: gzip` tag, about 20 MB instead of 77) and + without source maps, with `ar-io/ar-io-deploy` (pinned by commit), and moves the ArNS + name. Pushes to `dev` never touch Arweave. To try library mode on Arweave before a + release, run *Staging build to Arweave* (`staging-arweave.yaml`) by hand on a branch: it + uploads a staging build the same way, paid by the deploy wallet, with no ArNS name, and + puts the manifest ID in the job summary (`docs/LIBRARY_PAGES.md`). + Each upload then reads every path through turbo-gateway.com and ardrive.net with + `tool/warm_arweave_manifest.sh `, which also fails on a gateway sending gzip + bytes without the header. A new upload can 404 on a gateway other than + turbo-gateway.com for a while after it lands: that's propagation, not a broken deploy. ## Working alongside other agents diff --git a/docs/LIBRARY_PAGES.md b/docs/LIBRARY_PAGES.md index e3ef1e7f4c..2587e9c5cb 100644 --- a/docs/LIBRARY_PAGES.md +++ b/docs/LIBRARY_PAGES.md @@ -1,331 +1,414 @@ # Library pages -> **Status: PLAN.** Not built, and blocked on the drive state artifact landing -> first. A library page maps an ArNS name to an ArDrive drive or folder so the -> name opens a browsable page, public or private. Evidence is cited to file and -> line; gateway behaviour was measured against live gateways in October 2026. +> **Status: DESIGN v2, agreed 2026-10-21. Not built.** Build order, code seams +> and tests are in `docs/library-pages/IMPLEMENTATION_PLAN.md`. Supersedes v1 +> of this file (a plain HTML viewer with range reads), which is in git history. -## What this is for - -An owner points `phils-archive.ar.io` at a drive, or at one folder of it, and -anyone who opens that name gets a browsable page. A public drive opens with no -key. A private drive opens once the visitor supplies the drive key. - -The shipped app can assign an ArNS name to one file and to a manifest, so a name -can already point at one object. It cannot point at a library, because nothing -generates the page that lists one and nothing serves the catalog that page would -read. +A library page is a permanent web page that shows one ArDrive drive, or one +folder of it, to anyone who opens its link. It is ArDrive itself in a +read-only "library mode": the same file table, previews, search and downloads, +under the owner's title, icon, social card and cover. A public library opens +as it is; a private one opens with the drive key. ## Vocabulary -One term, one meaning, in this document and in the UI: - -| Term | Means | -|---|---| -| Library page | What a visitor opens at the name | -| Publish, republish | The action that writes it | -| Drive state artifact, or artifact | The published catalog the page reads (`DRIVE_STATE_ARTIFACT.md`) | -| Manifest | The Arweave path manifest that serves the page and its files | -| Name | The ArNS record pointing at that manifest | - -**Publish is not Share**, and the UI says so. Share hands someone a link to what -already exists, writes nothing to Arweave and costs nothing. Publish writes -permanent objects to Arweave, costs an upload, and cannot be undone. - -## It is built on the drive state artifact - -A library page invents no data format. The drive state artifact already carries -what the explorer renders: the drive, both entry tables, all three revision -tables and licenses (`DECISIONS.md` D2), gzipped, and encrypted under the -drive key when the drive is private. Its read path is already the one a static -page needs: resolve a name, check the artifact's owner, fetch bytes, decrypt, -read (`DRIVE_STATE_ARTIFACT.md` §4.1). The owner check is one lookup by -transaction id, the same attribution every ArFS entity gets (D16). - -Three consequences: - -- **No listing file.** An earlier revision of this plan proposed publishing a - separate JSON catalog beside the manifest. That duplicated the artifact and is - dropped. -- **Version history works in the page**, because D2 reversed to carry every - revision rather than the newest per entity. -- **Licenses, assigned ArNS names, pinned owners and thumbnails all travel**, so - the page renders what the explorer renders without asking a gateway anything - else. - -This feature also closes D6, which parked ArNS discovery as "not wired in v1". -A library page is the case §4.1 describes: a recipient handed a name and a key -opens a large drive without walking its history. - -## The shape - -Publishing writes two objects and one record: - -1. **The artifact**, the drive's catalog, as specified in its own document. -2. **The manifest**, whose index is the viewer, which maps one path to the - artifact, one path to the shared viewer code, and one path per published file - to its data item. -3. **The record**, pointing the chosen name at the manifest. - -The viewer itself is deployed once per app release and referenced **by -transaction id** from each page's manifest, never by a name. A manifest path may -point at any transaction, so every library page shares one copy of the viewer at -no extra cost, same-origin and immutable. A permanent page must not have a -mutable record in its critical path. - -Every link the page emits is relative, so the page carries no gateway hostname -and keeps working on any gateway that resolves the name, and under `ar://`. - -## Private library pages - -The manifest and the viewer are public. The artifact inside them is encrypted, -which is how a private drive's page works: - -- The visitor supplies the **drive key**, pasted or carried in the link exactly - as a shared private drive does today with `driveKey`. -- That key decrypts the artifact, and every file key derives from it through - `deriveFileKey()` (`lib/core/crypto/crypto.dart:119`), so the page decrypts - file bytes with WebCrypto, choosing AES-GCM or AES-CTR at the uploader's - boundary (`packages/ardrive_uploader/lib/src/constants.dart:3`, 100 MiB). -- **No new cryptography.** The page runs the derivation the app already ships - and the artifact's own envelope. Nothing in the upload path moves. - -**A private page's manifest paths must be opaque.** Manifest path keys are -plaintext inside a public JSON object, so publishing `Photos/medical/scan.jpg` -as a key publishes that structure even though the bytes stay encrypted. Private -pages key their paths by file id, and the readable tree lives only inside the -encrypted artifact. - -What the review step has to say, in the step itself rather than in a notice: - -- Publishing is permanent and cannot be withdrawn. -- A shared drive key opens the whole drive for ever, including files uploaded - after the publish. That is already true of sharing a private drive today. -- Clearing the record stops the name resolving. It does not unpublish anything. - -Wrapping the drive key under a passphrase, so a link alone is not the key, is a -later refinement rather than the mechanism. - -## How the page reads the artifact - -The viewer opens the artifact with SQLite compiled to wasm, which this -repository already ships and loads on every page (`web/sql-wasm.wasm`, 1.2 MB -raw, about 458 KB gzipped, loaded at `web/index.html:332`). DuckDB is the right -tool for parquet analytics over a global index and the wrong tool for opening -one drive's catalog. - -Gateway behaviour, measured in October 2026 against two AR.IO gateways in three -browsers, cross-origin: - -| Property | Measured behaviour | +| Term | Means | Shown to users as | +|---|---|---| +| Library page | What a visitor opens | Library page | +| Publish, republish, unpublish | The owner's actions | Publish library page, Republish, Unpublish | +| Snapshot | A drive state artifact (`DRIVE_STATE_ARTIFACT.md`), the catalog the page loads | Snapshot | +| Viewer bundle | One release of the ArDrive web build, uploaded to Arweave | never shown | +| Page manifest | The Arweave path manifest that is the page | never shown; its link is the page's link | +| Name | An ArNS name or undername pointing at a page manifest | Name | + +Publish is not Share. Share hands out a link to what exists, writes nothing and +costs nothing. Publish writes permanent objects to Arweave. + +## Decisions (all agreed) + +1. **The viewer is ArDrive in library mode, from the same build.** No second + UI. Every release is a viewer. +2. **Every publish assembles a fresh, static page manifest** that pins one + release's bundle by transaction ID. The bundle is never re-uploaded per + page. A page runs the release it pinned until it is republished. +3. **Release bundles are uploaded gzipped** with `Content-Encoding: gzip` + (`@ar.io/deploy --compress gzip`, action `ar-io/ar-io-deploy` v2.1.2). + This also shrinks the permaweb ArDrive at `ardrive.ar.io`. +4. **Snapshots stay always compressed.** The viewer loads the whole snapshot, + so there is no range-read threshold. Remove "Producers always compress until + Library pages ship" from `DRIVE_STATE_ARTIFACT.md` §2.7. +5. **Customization:** title, description, icon, social card, cover banner + (not a full-page background). +6. **Publish lives beside Share** in a drive's menu, a folder's menu and the + details panel. Not in the New menu. +7. **No analytics, no third-party requests** on a library page. +8. **Every page shows "Made with ArDrive · Open in ArDrive".** +9. **No per-file URLs** in v1. Files load by transaction ID from the serving + gateway. +10. **Republish only on request,** choosing an existing snapshot or making a + new one. +11. **Names:** the owner picks the main name or an undername. Unpublishing an + undername removes its record; unpublishing a main name points it at a + shared "no longer published" page. See [Names](#names): this whole part + waits on ArNS-on-Solana support in the app. + +## Measured facts this design rests on (2026-10-21) + +| Fact | Value | |---|---| -| Range requests | Work on `/{txId}`, a bundled data item, a manifest path and an ArNS path, single and multi-range | -| CORS | `Access-Control-Allow-Origin: *`, `range` allowed, headers exposed | -| A gzip-stored object plus a range | Returns a slice of the compressed bytes, which no browser can decode | -| Negotiated compression | Does not happen. No gateway compresses on request | -| A cold object | The gateway fetches the whole object before answering a range: 9 to 22 seconds measured | -| A warm object | About 70 ms per read | - -Five rules follow, and they are requirements rather than preferences: - -1. **Publish the artifact with no `Content-Encoding` tag.** ArDrive sets none - anywhere today, so this costs nothing to honour. -2. **Compress inside the envelope only below the range-read threshold.** A small - artifact is fetched whole and compression is free value. A large public one - is published identity so the page can read the pages it needs. Readers tell - the two apart by the payload's first bytes, so the choice needs no flag, and - it is permanent per artifact. -3. **Warm the cache at publish.** One full GET against the serving gateway after - upload, so the first visitor never pays the cold-miss cliff. -4. **Read in few large blocks.** The gateway's per-resource token bucket is keyed - `host:method:path` and shared by every reader of the same page, charged per - KiB with a one-token minimum, so hundreds of 4 KB reads are the wrong shape. -5. **Ignore `Content-Digest` on a partial response.** A gateway returns the - whole-object digest on a 206 and signs it, so verifying it always fails. The - page relies on the artifact's owner check, and on GCM for a private drive, - which is the trust model the artifact specifies anyway (D16). With no - signature over the whole payload, a range read is no less trusted than a - full one. - -The page also pins `x-arns-resolved-id` from the first response and reads the -rest against that id, because a name is mutable and `If-Range` is not -implemented. - -## The publish flow - -Three entry points open the same flow with a different preselected scope: a -drive's details panel and row overflow menu, a folder's, and one entry in the -New menu. Publish belongs beside Share, which already appears in all three -places (`lib/components/details_panel.dart:1341`, -`lib/pages/drive_detail/drive_detail_page.dart:777`, -`lib/drives_list/presentation/drive_actions_menu.dart`). - -The New menu gains one item and loses the temptation to add three: - -```text -Upload Files · Upload Folder · New Folder · New Note · New File Pin -──── -Publish… -──── -New Drive · Attach Drive -Advanced ▸ Import from Manifest · Create Manifest · New Snapshot +| `ardrive.ar.io` `main.dart.js`, as served today | 28.2 MB, no compression, even with `Accept-Encoding: gzip` | +| Same file from `app.ardrive.io` (Firebase, brotli) | 2.9 MB | +| `main.dart.js` gzipped | 27.5 → 4.0 MB | +| `canvaskit.wasm` gzipped | 6.4 → 2.6 MB; Chromium build 5.1 → 1.9 MB | +| `Content-Encoding` tag passthrough | turbo-gateway.com, ar-io.dev, ardrive.net, vilenarios.com send the header (raw and sandboxed routes); frostor.xyz decompresses itself. A gateway that has not indexed a new item yet may send gzip bytes bare (ar-io-node #964/#966, fixed on ar.io and Turbo gateways) | +| Release manifest | 469 paths, 49 KB | +| Staging build uploaded gzipped (phase 0) | 466 files without source maps, 77.3 MB → 18.8 MiB; 19 s; 0.2666 Turbo credits (about $1.58) for a first upload; later uploads pay mostly for `main.dart.js` (4.2 MB) | +| First visit to the gzipped bundle | 8.96 MB on the wire, Flutter view at 2.9 s (turbo-gateway.com and ardrive.net alike); the uncompressed production upload of the same day: 38.16 MB | +| ar-io.dev and gzip | Serves some gzipped items decompressed even when the browser accepts gzip (correct, larger). Served every path of a new Turbo upload only 1 h 27 min after it (404s until then, `AssetManifest.json` among them, so the app stopped at its splash), while turbo-gateway.com and ardrive.net served all of them at once. Charges for data past a free allowance: HTTP 402 with an x402 offer (USDC on Base Sepolia) after one evening of testing from one connection. It is the gateway of Turbo's **development sandbox** (`upload.services.ar-io.dev` is the Turbo SDK's `developmentUploadServiceURL`): uploads through that service show on ar-io.dev at once but are not permanent and no production gateway serves them; production uploads reach ar-io.dev only from the chain. Not a gateway for releases or library pages | +| ArDrive at a raw manifest URL | Boots: `https://turbo-gateway.com//` redirects to the sandbox subdomain and the app loads (no `` tag in `web/index.html`, all paths relative) | +| Snapshot of 41,767 files | 6.0 MiB gzipped, imported in 0.6 s | +| Requests the normal app makes on start | Also fonts.gstatic.com, upload.ardrive.io, plausible.io: library mode must stop these | + +## Architecture + +### What a page is + +A page manifest (`arweave/paths` 0.2.0) whose paths are: + +| Path | Points at | Uploaded by | +|---|---|---| +| `index.html` (index and fallback) | The release's `index.html` with the page's head written in | the publish | +| `library.json` | The page's settings and its snapshot's tags | the publish | +| `snapshot` | A drive state artifact | earlier, or the publish | +| `icon-32.png`, `icon-180.png`, `icon-512.png`, `social.jpg`, `cover.jpg` | The page's images, when set | the publish, or an existing transaction | +| every other path of the release manifest | The release's own files, by transaction ID | CI, once per release | + +A page manifest is about 52 KB, under Turbo's free 105 KiB. Its link is +`https:////` (the trailing slash matters), and a name can +point at it later. + +### `library.json` + +```json +{ + "version": 1, + "driveId": "c863be1f-a725-4554-9a9e-18268ed8a035", + "rootFolderId": "41520b33-28c1-4892-886b-b845a5709085", + "scope": { "type": "drive" }, + "privacy": "public", + "owner": "iKryOeZQMONi2965nKz528htMMN_sBcjlhc-VncoRjA", + "snapshot": { + "path": "snapshot", + "txId": "", + "tags": { "Drive-Id": "…", "Block-Start": "0", "Block-End": "1879136", + "Entity-Count": "6", "State-Version": "…" } + }, + "title": "Phil's Archive", + "description": "Photos, notes and the odd recording.", + "images": { "icon": "icon-512.png", "social": "social.jpg", "cover": "cover.jpg" }, + "publishedAt": "2026-10-21T18:02:11Z", + "viewer": { "release": "2.91.0", "manifest": "" } +} ``` -**"Publish Drive State" must not reach a user.** An artifact is how Publish -works, as a manifest is. Four engineering artifacts offered as four menu items -is how a user ends up choosing between Create Manifest, New Snapshot and Publish -Drive State. The drive state branch adds exactly that string today -(`lib/components/new_button/new_button.dart`), and it folds into Publish. - -The flow is one dialog with four steps, and the first carries most of the value: - -1. **Confirm what gets published.** Scope, file count, total size, and what is - left out with counts: hidden items, pending uploads, ghost folders, and names - that collide after path normalization. -2. **Choose the name.** The name itself and its undernames in one picker, with - the resulting URL built by `getAddressesFromArns()` - (`lib/arns/utils/arns_address_utils.dart`). -3. **Settings.** Title, a line of description, cover, accent, layout (list, grid - or gallery), default sort, and which columns show. -4. **Review and publish.** Cost and payment method, the permanence line, then - progress across the uploads and the record write. - -Preconditions are the artifact's own, reused rather than restated: an explicit -user action (D3), never from a sync that reported skipped entities, never above -the payload bound (D5), paid through whichever upload path the user already has -(D4), and hidden from a wallet that does not own the drive. - -The states after the first publish matter more than the flow: - -| Condition | What the UI shows | Action | -|---|---|---| -| Published and current | The URL, the published date, the file count | Copy, open, republish | -| Published and behind | How many files changed since the published date | Republish | -| Drive still syncing | That the local view is incomplete | Unavailable, with the reason | -| Pending uploads in scope | Which files are not yet confirmed | Unavailable, with the reason | -| No ArNS name on the wallet | What a name is and where to get one | Link out, never a dead end | -| Signed in with Ethereum or Solana | That a record write needs an Arweave wallet | Explain, rather than fail at signing | -| Not on web | Nothing | Hide the action | - -Those states derive from `DriveListItem` -(`lib/drives_list/domain/drive_list_item.dart`), which already carries -`hasBeenWalked`, `lastSyncedAt`, `isSyncing`, `lastSyncFailed` and -`hasUnreadChanges`. Publishing adds one fact, the published artifact and its -date, and derives the rest. - -## What the page shows - -A read-only library, not an editor: breadcrumbs and a folder tree, list, grid -and gallery views, client-side search and sort over the catalog it already -holds, and per row the name, size, type, modified date, a license badge where -`licenseTxId` is set, attribution where `pinnedDataOwnerAddress` is set, the -ArNS names a file carries from `assignedNames`, and version history, since every -revision travels in the artifact. - -Image rows use the thumbnail variants ArDrive already uploads -(`lib/models/tables/file_entries.drift:19`, `Variant` in -`lib/entities/file_entity.dart:213`). The footer names the artifact transaction -and the published date. The page makes no request to anything but the gateway -serving it and carries no analytics. Theme follows the ArDrive brand palette in -light and dark, as CSS variables in the viewer rather than imported from the app. - -## ArDrive specifics that have to be handled - -Ordered by damage. The first two were verified with a throwaway test against -`ManifestDataBuilder`; the rest are read off the code at the cited lines. - -| # | Finding | Evidence | What has to change | +- `scope` is `{"type":"drive"}` or `{"type":"folder","folderId":"…"}`. +- `snapshot.tags` copies every tag the importer reads from the artifact + (`DriveStateArtifactCandidate`), including `Cipher` and `Cipher-IV` for a + private drive. The viewer builds the candidate from these instead of + GraphQL. +- Nothing in `library.json` is secret. For a private drive it must not contain + any drive, folder or file name (the title is typed by the owner). + +### The page's `index.html` + +The release's `index.html`, with these written into `` (every value +HTML-escaped): + +- ``: the switch that puts + the app in library mode. +- ``, `<meta name="description">`, `<meta name="theme-color">`. +- `<link rel="icon" sizes="32x32">`, `<link rel="apple-touch-icon">`. +- `og:title`, `og:description`, `og:type=website`, `og:image`, + `twitter:card=summary_large_image`. `og:image` must be an absolute URL known + before `index.html` is uploaded, so it cannot use the page manifest's ID: + the manifest lists `index.html`, so it is built after it. The publish uploads + the social image first and points `og:image` at that transaction, + `https://<gateway>/<social-image-tx-id>` on the gateway the owner's app + uses (a reused image keeps its existing transaction). It stays that URL when + a name is added later, because `index.html` cannot change. With no social + image, `og:image` and `twitter:card` are left out. + +### How a page opens + +1. The gateway serves the manifest's `index.html`. +2. ArDrive starts, finds the meta tag, and enters library mode before any + service starts: no stored config, no login, no analytics, no Turbo, no + GraphQL, no ArNS, no HTML auto-reload, an in-memory database. +3. It reads `./library.json`. +4. File data comes from the gateway serving the page (the existing + `/ar-io/info` detection on the page's host). +5. A private library shows the unlock screen. +6. It fetches `./snapshot`, seeds the drive row from `library.json`, and + imports with the existing `DriveStateImporter`. +7. The explorer opens at the scope's root. No sync ever runs. + +**Rule, enforced by a test:** a library page makes requests only to the host +serving it and the gateway it detected, nothing else. + +### The viewer bundle + +- Production deploys, and staging builds uploaded by hand + (`staging-arweave.yaml`), use `--compress gzip`, leave out source maps, + and keep `index.html`'s relative paths. +- The app knows which release a new page pins through a config value, + `libraryViewerManifestSource`. A 43-character value is a manifest ID, used + as is. Anything else is an ArNS name, which the publisher resolves to the + manifest ID (`x-arns-resolved-id`) with a plain HTTP read through the + gateway. That needs no wallet, so it works before names do in the app (see + [Names](#names)). +- Production sets `ardrive`, the name the production deploy moves. Staging has + no name and pushes to `dev` never upload to Arweave, so the staging and + development configs also use `ardrive`: a page published from staging pins + the current production release. To try a viewer built from a branch, run + *Staging build to Arweave* (`staging-arweave.yaml`) and put the manifest ID + from its job summary in the stored config (`flutter.config` in local + storage). The phase 1 test-page tool takes a manifest ID directly. +- CI warms the bundle after each deploy by reading every file once through + turbo-gateway.com and ardrive.net. + +## Names + +**ArNS moved to Solana** (`@ar.io/sdk` 4.x). Names are Metaplex Core assets +owned by Solana keys; every record change needs a Solana signer. Names held by +Arweave wallets before the move sit in an escrow until claimed into a Solana +wallet (`ANTEscrow`, Arweave-attested claim). The app's own name features are +stubbed (`packages/ario_sdk/web/src/index.js`: "not yet implemented for +Solana"), so "Assign name" does not work in 2.90 either. + +So library pages ship in two halves: + +- **Without names (phases 1 to 3):** a page is its manifest link. The owner + copies it, or points a name at the manifest ID by hand with any ArNS tool. +- **With names (phase 4):** the name step, unpublish, and the main-name + placeholder arrive with ArNS-on-Solana support in the app (a Solana wallet + connection, listing names the wallet owns or controls, setting and removing + records). That is its own project, and library pages consume it. + +## UX: the visitor + +All of it is the existing explorer (`DriveDetailPage`, the data table, the +details panel, previews, downloads, breadcrumbs, search), framed by library +chrome instead of account chrome. + +### Loading screen + +Centred on the page background: the library's icon (or ArDrive's), the title, +a thin progress bar, and one line that changes with the step: "Opening +library…", "Loading files…" (snapshot download, with MB of total when the size +is known), "Almost there…" (import). The title and icon come from +`library.json`, so this screen appears after one small request. + +### Unlock screen (private libraries) + +Same frame as the loading screen. Below the description: "This library is +private. Paste the drive key you were given to open it." A `Drive key` field +(`ArDriveTextFieldNew`, obscured, show/hide toggle), an **Open library** +primary button (disabled until the field is non-empty), and in small text "The +key stays in this browser." A key in the link (`#/?driveKey=…`) skips this +screen. A rejected key shows "This key doesn't open this library." under the +field, and the field keeps its text for correction. A typed key is never +written to the address bar. + +### Error screens + +One frame, a heading, one sentence, one action: + +| Case | Heading | Sentence | Action | |---|---|---|---| -| 1 | Hidden folders are published. Only a file's own hidden flag is honoured, and hiding a folder does not cascade. Verified: a hidden folder's `notes.txt` published as `secret/notes.txt`. | `manifest_data.dart:127`, `drive_queries.drift:35`, `folder_node.dart:67`, `hide_bloc.dart:72`, contrary claim at `create_manifest_form.dart:987` | Prune hidden folders and drives recursively from both the manifest and the published set, report the count, fix the form's claim | -| 2 | Path normalization drops files. Verified: `my file.txt` and `my_file.txt` produce one path carrying the second file's id. | `manifest_data.dart:103` | Give each colliding file its raw path, which gateways resolve because they decode a request before matching it. Never percent-encode a key: that would break every one of them. Fixed in #2265 | -| 3 | The index points at an arbitrary file when the folder has no `index.html` | `manifest_data.dart:129` to `:140` | A library page always sets the index to the viewer | -| 4 | The name picker drops the name itself and labels the apex "No undername" | `assign_name_bloc.dart:92` and `:118`, `assign_name_modal.dart:649` | Offer the apex in both flows, labelled as the name | -| 5 | Undername URLs use an underscore: `library_phils-archive.ar.io` | `get_literal_arns_record_name.dart`, host at `constants.dart:71` | Show the resolved URL rather than a dot-separated guess | -| 6 | Republishing is not instant: the record TTL is 900 seconds | `arns_record.dart:8` | Say so in the done state, and do not claim success at the name before the record reads back | -| 7 | Publishing from a partially read drive would publish a partial library | D3, and the warning already in `drive_state_creation_modal.dart` | Reuse the artifact's precondition rather than writing a second one | -| 8 | Pending uploads produce dead rows | `hasPendingFilesOnTargetFolder()` | Keep the gate, and name the files in the confirm step | -| 9 | Ghost folders have no metadata and are a normal state | `lib/sync/domain/ghost_folder.dart`, `folder_entries.drift:11` | Exclude them, and list them among the exclusions | -| 10 | Pinned files point at another wallet's transaction | `file_entries.drift:21` | Publish them, show the attribution | -| 11 | The manifest is an ArFS file inside the drive, so each page adds a file the user browses | `existingManifestFileId`, `manifest_repository.dart:118` | Mark library page manifests so the explorer presents them as the page | -| 12 | The assign action is web-only | `details_panel.dart:1603` | Hide publishing off web, keep the published state readable | -| 13 | Two separate gates block private drives | `new_button.dart:339`, `create_manifest_cubit.dart:62` | Change both in the private phase, routing private publishing through its own flow | -| 14 | A record write needs an Arweave signer, and `ProfileType` has two values | `arns_repository.dart:126`, `profile_types.dart` | Explain unavailability for Ethereum and Solana sign-ins rather than failing at signing | -| 15 | Drive scope is folder scope | `create_manifest_cubit.dart:82` | One code path, two entry points, recording the scope so republish does not widen it | - -Four findings from the earlier revision are resolved by building on the -artifact rather than a listing file: empty folders survive because folder rows -travel, licenses render because the `licenses` table travels, assigned names -render because `assignedNames` travels on the file rows (D10), and version -history renders because every revision travels (D2). - -## What breaks, and for whom - -A published page is a permanent public artifact supported for as long as Arweave -exists. Old pages keep running old viewers, so a viewer bug ships for ever and -is fixed only for pages republished afterwards. That argues for a small viewer -and for pinning its transaction per publish. - -A private page moves a security boundary from a wallet to a key the owner hands -out. Neither the artifact nor the ciphertext can be withdrawn. The UI has to be -honest at the moment of commitment rather than in a help page. - -A stale page misleads visitors in a way a share link does not, because a share -link reads live state. The published date and the republish prompt are part of -the feature rather than polish. - -## Scope - -1. **Land the artifact.** D12 accepted as the container, D5 re-derived, the - entity specified in `ar-io-docs`. Nothing here ships first. -2. **Public library pages.** Viewer, manifest, naming, publish and republish, - and findings 1 to 3 and 5 to 15. Done when a public drive and a public folder - each resolve at an owned name and the excluded counts match what is published. -3. **Private library pages.** Opaque paths, drive key unlock, in-page decryption - for both ciphers, and findings 12 to 14. Done when a private folder opens in a - clean browser profile and no file name appears in the manifest or in any tag. -4. **Passphrase unlock, and the full app as a second tier.** The app already - deploys itself to Arweave under a name through `ar-io/ar-io-deploy@v1` - (`.github/workflows/production.yaml:59`), so once it can hydrate a drive from - an artifact, a deployed copy becomes the heavyweight option beside the viewer. - -## Decisions needed before building - -- Whether the default offered name is the apex or an undername. -- What unpublish means, given that clearing the record stops resolution while the - data stays readable by transaction id for ever. -- The size threshold at which a public artifact is published identity rather than - compressed: measured with the real page, starting from 16 MiB (D13). -- Whether the viewer source lives in `assets/` here or in its own repository. -- Whether republish happens automatically after an upload into a published - folder, as AutoDeploy does for manifests, or stays explicit. - -## Tests the implementation has to carry - -- A hidden folder holding visible files publishes none of them, and the excluded - count reports them. -- Two files whose names normalize to one path both appear, deterministically - ordered, and the collision is reported before publishing. -- A private publish leaks no file name, folder name or path fragment into the - manifest JSON or any transaction tag. -- The viewer opens an encrypted artifact with the drive key and decrypts a file - below and a file above the 100 MiB GCM boundary. -- Publishing is unavailable while a drive syncs and available once it settles, - asserted on cubit states rather than on a call count. -- Republish writes a new revision of the same ArFS file and points the record at - the new manifest. -- The apex record is offered for a name that has undernames. -- The viewer renders from the artifact with no GraphQL request at all. - -Each test earns its place by failing when the rule it covers is broken on -purpose. - -## References - -- `docs/DRIVE_STATE_ARTIFACT.md` and `docs/drive-state/DECISIONS.md`, the format - this feature reads. -- `lib/entities/manifest_data.dart` and `lib/manifest/domain/manifest_repository.dart`, - the manifest and its uploader. -- `lib/arns/`, the record picker, repository and address helpers. -- `lib/core/crypto/crypto.dart` and `docs/private_drive_kdf_reference.dart`, key - derivation for the private phase. -- `docs/SHARE_LINKS.md`, the other way an ArDrive entity reaches someone else. +| `library.json` missing or unreadable | This library can't be opened | Its settings are missing or damaged. | Open ArDrive | +| Snapshot can't be fetched | This library can't load right now | The gateway didn't return its files. Try again in a minute. | Try again | +| Snapshot rejected by the importer | This library can't be opened | Its data didn't pass ArDrive's checks. | Open ArDrive | +| Viewer older than `library.json`'s version | This library needs a newer ArDrive | Ask its owner to republish it. | Open ArDrive | + +### The library frame (desktop) + +- **Sidebar** (replaces drives list, New button and profile): the icon (48 px), + the title, the description, "Published <date>", then at the bottom "Open in + ArDrive" (link) and "Made with ArDrive" (muted), then the existing collapse + control. +- **Top bar:** the existing search field ("Search this library…"), the theme + toggle. No login, profile card, sync indicator or New button. +- **Cover band:** on the library's root only, above the breadcrumbs: the cover + image, cropped to fill, 200 px tall on desktop, with a bottom scrim and the + title plus "N files · size · published <date>" over it. Without a cover, the + band is the title line on the normal background. On subfolders the band is + gone and the breadcrumbs lead. +- **Breadcrumbs** start at the scope's root; the root crumb is the title. +- **Data table, details panel, previews:** unchanged. +- **Row and toolbar actions offered:** Download, Download folder (zip), Preview + in the panel, More info, and **Copy link**, which copies the library's own + link to that folder or file (`<page link>#/folders/<id>?file=<id>`). + Everything else is hidden: Share (replaced by Copy link), Rename, Move, Hide, + License add/update, Pin, Create snapshot, Sync, Full resync, Export, Attach. +- **Never shown:** hidden items or anything inside a hidden folder, ghost + folders, failed or pending uploads, anything outside a folder library's + folder, the "Show hidden" toggle. + +### The library frame (phone) + +ArDrive's mobile layout. The app bar shows the icon and title; the drawer holds +the sidebar's content. The cover band is 120 px tall and crops to its centre. +No FAB. + +### Copy + +Library chrome strings are new keys in `app_en.arb`. Dates use the app's +existing date formatting. + +## UX: the owner + +### Entry points + +"Publish library page" with a globe icon, beside Share in: the drive's overflow +menu (desktop and mobile), a folder row's menu, the details panel toolbar for a +drive or folder. Shown only when: web, signed in, the wallet owns the drive. +Once a scope is published, the item reads "Library page…" and opens the +management section instead. + +### The publish dialog + +One `ArDriveStandardModal`-style dialog, 560 px wide, a step indicator ("Step 2 +of 5"), Back and Next in the footer, Cancel always available. Closing midway +asks "Discard this library page?" only once something has been uploaded. + +**Step 1 · What gets published.** Scope as two radio rows when opened from a +drive with a folder selected, otherwise fixed and named. Then: files, folders, +total size; and "Left out:" with counts for hidden items, items in hidden +folders, pending uploads, ghost folders, each with a one-line reason. Blocking +states replace Next with the reason: "This drive is still syncing", "N uploads +in this folder aren't confirmed yet" (with the file names, collapsible), "This +drive hasn't been fully read on this device. Sync it first." + +**Step 2 · Snapshot.** "Which snapshot should the page show?" Radio rows, newest +first, for every drive state snapshot of this drive made by its owner: date, +file count, size, and "N changes since" (revisions after its `Block-End` in the +local database). Last row: "Create a new snapshot · Includes every change up to +now · <cost>". Below: "Snapshots in the older format can't be used." Over 100 +MiB: the row is disabled with "Too large for a library page". With no +qualifying snapshot, "Create a new snapshot" is preselected and the only +option. + +**Step 3 · Look.** Fields with live preview on the right (a social-card preview +and a browser-tab preview with the icon): + +| Field | Control | Rules | +|---|---|---| +| Title | text, 60 chars | Required. Prefilled with the drive or folder name for a public drive, empty for a private one | +| Description | text, 160 chars, counter | Optional | +| Icon | image picker | Square, at least 512 × 512, PNG/JPEG/WebP/SVG, under 2 MB. Makes 32, 180, 512 px PNGs | +| Social card | image picker | 1200 × 630. Cropped to 1.91:1, re-encoded JPEG, aiming under 100 KiB | +| Cover | image picker | 1600 × 400. Cropped to 4:1, re-encoded JPEG, aiming under 100 KiB. Hint: "Keep text and faces in the middle; phones crop the sides" | + +The image picker is a thumbnail plus "Change" opening a menu: **Upload an +image**, **Choose from this drive** (public drives only; lists image files), +**Use a transaction ID** (validated: the gateway returns an `image/*` content +type and the size rules above), **Remove**. For a private drive, "Choose from +this drive" is disabled with "Files in a private drive are encrypted. Upload +the image instead." A too-small or wrong-type image explains what is wrong and +what size to use. + +**Step 4 · Name.** Until phase 4: a short panel: "Your library page will have +its own link. You can point an ArNS name at it later." No input. In phase 4: +the name picker (see [Names](#names)). + +**Step 5 · Review and publish.** A summary (scope, snapshot, title, link or +name), the cost lines (new snapshot if any; images; page files: "Free") with +the existing payment cards, then, in the step itself, not a tooltip: + +- "Publishing is permanent. It can't be undone or deleted." +- Private only: "Anyone with the drive key can read the whole drive, including + files added later, folders outside this one, and hidden items." +- Private only: "The title, description and images are public." + +The button is **Publish library page**. + +**Progress.** The same dialog lists the steps with spinners and ticks: +snapshot (if new), images, page settings, page, page manifest, warm-up. A +failed step shows its error and **Try again**, which resumes from that step +(already uploaded items are reused, not re-paid). + +**Done.** "Your library page is published." The link with **Copy** and +**Open**. Private: two copy buttons, "Copy link" and "Copy link with key", +the second with "Anyone with this link can read the whole drive." A note: "A +new page can take a minute to load the first time." + +### Management section + +In the details panel of a published drive or folder, a "Library page" section: +status chip (Live), the link with Copy and Open, "Published <date> from the +<date> snapshot", "N changes since then", **Republish**, and in phase 4 +**Unpublish**. States: + +| Condition | Shows | Action | +|---|---|---| +| Live and current | Link, dates | Copy, Open, Republish | +| Live and behind | "N changes since then" | Republish | +| Drive syncing | "Syncing; republish when it's done" | Republish disabled | +| Pending uploads in scope | "N uploads aren't confirmed yet" | Republish disabled | +| Not on web | Link and dates only | none | + +Published pages are found from the owner's own uploads by tags (below), so the +section shows on any device. No database change. + +**Republish** opens the dialog prefilled from the page's current +`library.json`, with scope fixed, starting at the snapshot step. It writes a +new page manifest; in phase 4 it also moves the name. + +### Tags + +Every library upload carries `App-Name: ArDrive-Web`, `App-Version`, and +`Type`: `library-page` (manifest), `library-config` (`library.json`), +`library-index` (`index.html`), `library-asset` (images). The manifest also +carries `Drive-Id`, `Library-Scope` (`drive` or the folder ID) and +`Library-Version: 1`. Discovery: owner = wallet, `Type = library-page`, +`Drive-Id`. + +Images, `library.json`, `index.html` and the manifest are plain public data +items, never ArFS entities: they must be public for a private drive, and they +must not appear as files in the drive. + +## Private libraries + +- The key travels like a private drive link's, `#/?driveKey=…`, in the fragment + only. Typed keys are never written to the URL. +- The snapshot and files decrypt in the browser with the existing derivation. +- Nothing about the drive's contents is outside the encrypted snapshot: no + manifest file paths, no names in `library.json` or tags. +- The review step states the three facts above. + +## Phases + +| Phase | Ships | Done when | +|---|---|---| +| 0 · Compressed bundle (#2343) | `--compress gzip` in staging and production deploys; a staging channel | A staging upload loads through turbo-gateway.com and ardrive.net with every JS/wasm file carrying `Content-Encoding: gzip`; every path loads through ar-io.dev once it has fetched the upload (it may serve gzipped items decompressed, which is valid); first-visit transfer is measured | +| 1 · Library mode | The viewer, public libraries, a script that builds a test page by hand | A hand-built page at a raw manifest URL opens a public drive and a public folder, with no request outside the serving host and gateway | +| 2 · Publish (public) | Dialog steps 1, 2, 3, 5; management section; republish; tag discovery | A public drive and folder publish from the app; republishing with an existing snapshot and the same images costs nothing; excluded counts match the page | +| 3 · Private | Unlock screen, encrypted snapshots, private rules | A private folder opens in a clean profile with the key, and no drive, folder or file name appears in the manifest, `library.json`, `index.html` or any tag | +| 4 · Names | Step 4, unpublish, placeholder page | Needs ArNS-on-Solana in the app first | + +Phase 1 must reach a production release before any real page works, because a +page runs the release it pins. + +## Risks + +- **A gateway serving gzip bytes bare** breaks a page on that gateway. Load + each deploy through every gateway that matters before switching production. +- **Phones and large snapshots:** a snapshot near the 100 MiB bound lives in + memory. Measure in phase 1; above a size to be found, step 2 warns "This + library may be slow on phones." +- **Old pages run old code for ever.** Library mode must stay narrow (one host, + one gateway) so pages age well. +- **Fallback fonts:** Flutter fetches missing-glyph fonts from + fonts.gstatic.com. Library pages bundle them: the no-third-party rule has no + exceptions. If Flutter 3.19 cannot load them from the page, that blocks + phase 1 and goes to Phil. diff --git a/docs/library-pages/IMPLEMENTATION_PLAN.md b/docs/library-pages/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000000..aa78b8640c --- /dev/null +++ b/docs/library-pages/IMPLEMENTATION_PLAN.md @@ -0,0 +1,266 @@ +# Library pages: implementation plan + +The design is `docs/LIBRARY_PAGES.md` (v2, agreed). This file is how to build +it: phases in order, the code seams each phase touches, the tests that prove +it, and the rules for testing against real services. Read the design first, +then `CLAUDE.md`, `docs/DRIVE_STATE_ARTIFACT.md` §2 and §4, and +`docs/WEB_E2E_TESTING.md`. + +Each phase is one pull request to `dev` (phase 2 may be two). Do not start a +phase before the previous one is merged. + +## Ground rules + +- **Credentials.** The test wallet's keyfile path and password live only in + `~/.config/ardrive-e2e/env`. Never print, copy, commit or upload them, or + echo the keyfile path. Never dump DOM input values in a browser script (the + password field is one of them). Drive keys are secrets too: write them to + `chmod 600` files, never to output, and crop screenshots that show a key + field. +- **Spending.** The test wallet (`iKryOe…ncoRjA`) has Turbo credits and may be + used for phase 0 and phase 1 uploads, as Phil authorized on 2026-10-21. + Report the cost of every paid upload. Prefer free sizes (under 105 KiB) + wherever a test allows. Anything else needs Phil's go-ahead first. +- **Merges and force-pushes are Phil's.** A guard blocks both. Open PRs, run + CI, report; Phil merges. +- **Browser tests** use `tool/web_e2e/` with your own `E2E_DIR`, `CDP_PORT` + and app port; other sessions run the same tools on this machine. Kill only + PIDs you recorded. On WSL, run the harness from a Linux-side copy (see + `docs/WEB_E2E_TESTING.md`, Known quirks). +- **Flutter on this machine** is the Windows 3.19.6 SDK via + `/mnt/c/Windows/System32/cmd.exe /c "C:\Users\phili\fvm\versions\3.19.6\bin\flutter.bat …"`. + Commit with `LEFTHOOK=0`. Run `flutter analyze` without filtering its + output, and `dart format` only the files you changed. + +## Phase 0: compressed viewer bundle + +Goal: every release's web build is on Arweave gzipped, and a staging channel +exists to test library mode before a release. + +1. **Staging build to Arweave, on request.** A manual workflow + (`.github/workflows/staging-arweave.yaml`, `workflow_dispatch` only) builds + the chosen branch with the staging config and uploads it with + `--compress gzip` and no ArNS update. Pushes to `dev` keep deploying + staging.ardrive.io and nothing else (Phil, 2026-10-22: no Arweave spend per + push). The action has no upload-only mode (it always runs `deploy` with + `arns-name`), so the workflow runs the CLI's `upload` command at an exact + version. Record the manifest ID in the job summary. +2. **Production deploy.** In `.github/workflows/production.yaml`, pin + `ar-io/ar-io-deploy` (v2.1.2, by commit), add `compress: gzip`. Keep its + ArNS inputs. + Source maps: add `compress-exclude` only if needed; better, delete + `build/web/*.map` before the deploy step (they are not needed on Arweave). +3. **Warm-up.** After each deploy, a step reads every path of the new manifest + once through turbo-gateway.com and ardrive.net + (`tool/warm_arweave_manifest.sh`). `ar.io` itself is the ar.io website, not + a gateway; ar-io.dev charges for data past a free allowance (x402), so it + is checked by hand, not warmed. +4. **Verify by hand before merging:** run the staging deploy once (or locally + with `npx @ar.io/deploy upload --compress gzip` and the test wallet), then: + - every `.js` and `.wasm` path returns `Content-Encoding: gzip` through + turbo-gateway.com and ardrive.net, on the sandboxed route; + - every path loads through ar-io.dev once it has fetched the upload. It + may serve gzipped items decompressed, which is valid, so it is checked + for loading, not for the header; + - the app boots at `https://turbo-gateway.com/<manifest>/` in a fresh + headless Chromium, and total transfer is under 10 MB (today: 37.6 MB); + - record cost, transfer size and time in the PR. + +Done when: a compressed staging bundle boots through turbo-gateway.com and +ardrive.net, and through ar-io.dev once it has fetched the upload. +Production's change merges with it, and Phil merges. + +## Phase 1: library mode (public) + +Goal: a hand-built page at a raw manifest URL opens a public drive and a +public folder, read-only, in library chrome, with no request outside the +serving host and its gateway. + +### 1.1 Detection and start-up (`lib/library/`, `lib/main.dart`) + +- New `lib/library/` feature, laid out like `lib/drive_state/` + (`data/`, `domain/`, `presentation/`). +- `LibraryMode` (domain): reads `<meta name="ardrive-library">` from the + document once, before `_initializeServices()`. Web only; always off + elsewhere. Unit-test the parser with a stubbed reader. +- `LibraryConfig` (domain): parses `library.json` (design §`library.json`). + Reject unknown `version` above what this build supports with a typed error + (drives the "needs a newer ArDrive" screen). Unit tests for every field, + missing fields, private without cipher tags, and HTML in title/description + (kept as plain text). +- In `main.dart`, when library mode is on: + - Config: do not read or write stored config (`ConfigFetcher` must not touch + `localStorage` key `config`). Start from the flavor asset, then set + `arweaveGatewayForDataRequest` from the AR.IO detection on the page's own + host (`ario_gateway_detector.dart`, already splits the hostname), without + caching the result. Force `autoSync` off. + - Database: `Database(openLibraryConnection())`, a new opener in + `lib/models/database/web.dart` that loads `sqlite3.wasm`, registers an + `InMemoryFileSystem` (see `_sessionFileSystem`), sets `webSqlite`, and + returns a `WasmDatabase` with no persistence. No Web Lock, no worker, no + migration from sql.js. The drive state importer must work against it + (it uses `webSqlite` / `ATTACH`). + - Skip `refreshHTMLPageAtInterval`. + - Skip the Solana RPC re-init. +- Analytics: add a static `PlausibleEventTracker.disabled` flag checked in + `_track` (and any other send path); set it in library mode. Test that no + event is sent when disabled. + +### 1.2 Loading the library (`lib/library/data`, `lib/library/presentation`) + +- `LibraryLoader` cubit, states: `loadingSettings`, `needsKey` (phase 3), + `loadingSnapshot(bytes, total?)`, `importing`, `ready(LibraryConfig)`, + `failed(LibraryFailure)` with the four failures in the design's error table. +- Steps: + 1. GET `./library.json` (relative to `Uri.base`). + 2. Seed the drive: insert a `drives` row from `library.json` (`driveId`, + `rootFolderId`, `owner`, `privacy`, name = title for public or `''` for + private, `lastUpdated` = epoch so the snapshot's drive metadata wins) and + the root folder placeholder, the way `DriveDao.writeDriveEntity` does. + 3. GET `./snapshot` with progress. + 4. Build `DriveStateArtifactCandidate(txId, ownerAddress: owner, + tags: snapshot.tags)` and call `DriveStateImporter.import(candidate, + body, protection: DriveStateProtection.forDrive(privacy, key), + expectedOwnerAddress: owner)`. A rejected result maps to "Its data didn't + pass ArDrive's checks", and the outcome reason is logged. +- The importer runs its own owner check against `candidate.ownerAddress`; + that is the owner `library.json` names, which is as trustworthy as the page. + Do not add a GraphQL owner lookup. +- Tests: a cubit test per state transition with fakes; an integration test + that imports a real fixture artifact (see `test/drive_state_sqlite/` + fixtures) (the golden fixtures from #2297, under `test/drive_state_sqlite/` and `test/drive_state/`) into an in-memory database through the loader and reads the + folder tree back. + +### 1.3 Router and shell (`lib/pages/app_router_delegate.dart`, `lib/app_shell.dart`) + +- In library mode the router has exactly two outcomes: the `LibraryLoader` + screens until `ready`, then the explorer for the drive at the scope's root. + Sign-in, get-started, drives list, shared-file and `/view/` routes map to + the library root. Deep links inside the library (`#/folders/<id>`, + `?file=<id>`) are honoured when the folder is inside the scope, otherwise + they land on the root. +- Never start a sync: guard `SyncCubit.startSync`, `startSyncForDrive`, + `syncMetadataOnly` and the periodic timer with library mode (return without + emitting `SyncInProgress`). `openDriveAsVisitor`'s sync-on-open must not run. + Assert in a test that `SyncRepository` is never called in library mode. +- `LibraryShell`: the design's sidebar (icon, title, description, published + date, Open in ArDrive, Made with ArDrive), top bar (search, theme), cover + band on the root, and the mobile variants. Reuse `ArDriveTheme`, + `ArDriveTypographyNew`, existing sidebar and top bar widgets with the + account parts removed; do not fork the explorer. +- Explorer behaviour in library mode: + - hide hidden items and their descendants, ghost folders, and failed or + pending uploads (filter at the query or mapper level, not by hiding rows + after layout); + - breadcrumbs and search stop at the scope root; + - row and toolbar actions: Download, Download folder, Preview, More info, + Copy link (new: library link to the folder or file); hide everything else + listed in the design; + - no New button, FAB, profile card, login, sync indicator, "Show hidden". +- "Open in ArDrive" opens `https://app.ardrive.io/#/drives/<id>` (the 2.90 + visitor flow) in a new tab. + +### 1.4 Requests + +- Run the app in library mode in headless Chromium and log every request's + host. Allowed: the page's host and the detected gateway host. Expected + offenders to remove: `plausible.io`, `upload.ardrive.io` (a Turbo call at + start), `fonts.gstatic.com` (Flutter font fallback). Fonts are bundled, with + no exception: point `fontFallbackBaseUrl` in the page's Flutter + configuration at a bundled `fonts/` path. If Flutter 3.19 has no way to do + that, stop and bring it to Phil rather than allowing the host. + +### 1.5 Test page tool (`tool/library_page/`) + +A Node script, `build_test_page.mjs`, that builds a page by hand from: +a release manifest ID, a drive ID and root folder ID, an artifact transaction +ID (read its tags from the gateway's GraphQL once, here only), a title and +description. It writes `library.json` and the page `index.html` (head +injection, escaped), builds the page manifest (release paths + page paths, +index and fallback `index.html`), uploads the three small files with +`@ardrive/turbo-sdk` using the test wallet, and prints the page link. Every +upload is under 105 KiB, so it is free. This tool is also the reference +implementation for phase 2's Dart publisher: keep their outputs +byte-comparable (a golden test in phase 2 compares them). + +Snapshot for testing: create one for **New Test Drive** (public, +`c863be1f-a725-4554-9a9e-18268ed8a035`, 3 files) with Create snapshot on +staging, which is free at that size. For the folder case use a folder inside +`UAT-TESTONLY-DELETEME` (`e9b05236-…`). + +### 1.6 Verify + +- Build the branch for staging, deploy it to Arweave compressed (phase 0's + path), build a test page pinning that bundle, and open it in a fresh + profile: public drive, public folder, phone width, dark mode, a preview, a + download, Copy link, search, a deep link outside the scope. +- Network log shows only the allowed hosts. +- Record first-visit transfer, time to the file list, and import time. +- Measure a large public drive (ArConnect Archives, 711 files, or larger) on a + throttled 4G profile for the phones risk. + +## Phase 2: publish (public) + +- `LibraryRepository` (data): find published pages by tags (owner, `Type = + library-page`, `Drive-Id`) through the existing GraphQL client; read a + page's `library.json` through the gateway. +- `LibraryPublisher` (domain): the Dart version of the phase 1 tool. Inputs: + scope, chosen snapshot (or "new"), look settings, images. Steps: create + snapshot if asked (existing `DriveStateCreationService`), process images + (`packages/ardrive_utils/lib/src/generate_thumbnail.dart` already uses + `package:image`; crop and re-encode to the design's sizes), resolve the + viewer release (`libraryViewerManifestSource` config: a manifest ID as is, + or a name read through the gateway for its `x-arns-resolved-id`; then GET + `/raw/<id>`), upload the images, then write `library.json`, `index.html` + (its `og:image` names the social image's transaction) and the manifest, + warm up. Every upload goes + through the existing Turbo upload path (`ArDriveUploader` data items, the + user's payment choice). Resumable: keep uploaded transaction IDs per step + for the dialog's Try again. +- Snapshot list: `GraphQLDriveStateDiscovery` for the drive, owner-filtered; + "changes since" from the local database (revisions with block height above + `Block-End`). Refuse above 100 MiB (D5). +- UI: entry points, the five-step dialog (step 4 shows the "link now, name + later" panel), progress, done, the management section, republish. Copy and + layout exactly as the design specifies. +- Tests: golden test that the Dart publisher's `library.json`, `index.html` + and manifest equal the phase 1 tool's for the same inputs; escaping test; + preconditions on cubit states (syncing, pending uploads, not fully read); + excluded counts against a fixture drive; republish keeps scope and writes a + new manifest; image processing sizes and byte budgets. +- e2e: publish New Test Drive and a UAT folder from staging with existing + snapshots (free), open both links in a fresh profile, republish, check the + management section's states. + +## Phase 3: private + +- Unlock screen and `needsKey` state; key from `#/?driveKey=` or typed; typed + keys never reach the URL; wrong key keeps the text and shows the error. +- Import with `DriveStateProtection` for a private drive; put the key in + memory (`DriveDao.putDriveKeyInMemory`) so previews, downloads and + thumbnails decrypt. +- Publish rules: title empty and required; "Choose from this drive" disabled + for images; the three review-step statements; "Copy link with key". +- Tests: no drive, folder or file name appears in `library.json`, + `index.html`, the manifest or any tag (scan every byte the publisher + uploads); decrypt a file below and above the 100 MiB GCM/CTR boundary in + library mode; wrong key and missing key screens. +- e2e: Private Drive Test (`7cea4056-…`) published from staging; open in a + fresh profile with the key in the link and typed; reload behaviour. + +## Phase 4: names + +Blocked on ArNS-on-Solana support in the app (Solana wallet connection, +listing names a wallet owns or controls, setting and removing undername and +base-name records with `@ar.io/sdk` 4.x). When that exists: step 4's picker, +unpublish (remove an undername; point a main name at a shared "no longer +published" page deployed with each release), republish moving the name, +and TTL 300 s. `og:image` keeps pointing at the social image's transaction. + +## Updating the docs as you go + +- Phase 0: `CLAUDE.md` Deployment section (compressed Arweave deploys, staging + channel). +- Phase 1: `DRIVE_STATE_ARTIFACT.md` §2.7, drop "until Library pages ship". +- Each phase: mark its row in the design's Phases table as shipped, with the + PR number. Record measurements in the design's facts table. diff --git a/tool/warm_arweave_manifest.sh b/tool/warm_arweave_manifest.sh new file mode 100644 index 0000000000..f44c19d25b --- /dev/null +++ b/tool/warm_arweave_manifest.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Reads every path of an Arweave path manifest once through each gateway, so +# the first visitor after a deploy is not the one who makes the gateway fetch +# and index 466 files. Also checks what a browser would get: every path must +# answer 200, and no gateway may send gzip bytes without +# `Content-Encoding: gzip`, because the deploy uploads them gzipped and a +# browser handed them bare cannot run them. A gateway may instead decompress +# an item itself (ar-io.dev does for some); that works, at a larger transfer, +# so the summary counts the JavaScript and wasm paths served that way. +# +# tool/warm_arweave_manifest.sh <manifest-id> +# +# GATEWAYS (default "turbo-gateway.com ardrive.net", Turbo's and ArDrive's) +# and JOBS (default 4) override the gateways and the number of parallel +# requests. ar-io.dev is left out on purpose: it is the gateway of Turbo's +# development sandbox, sees production uploads only once they are on chain +# (over an hour), and charges for data past a free allowance (HTTP 402). Requests go through the sandboxed route +# (`https://<gateway>/<id>/<path>`, which redirects to the manifest's own +# subdomain), the route a library page is served on. +# +# Exits 1 if any path failed on any gateway; prints one line per failure, with +# curl's error when there was no response, and a summary per gateway. + +set -euo pipefail + +id="${1:-}" +if [[ ! "$id" =~ ^[A-Za-z0-9_-]{43}$ ]]; then + echo "usage: $0 <manifest-id>" >&2 + exit 2 +fi + +GATEWAYS="${GATEWAYS:-turbo-gateway.com ardrive.net}" +JOBS="${JOBS:-4}" + +# Turbo serves a new upload before it is mined, so read the manifest there. +manifest="$(curl -fsSL --retry 5 --retry-all-errors --retry-delay 5 \ + "https://turbo-gateway.com/raw/$id")" + +# Each path segment URL-encoded, so no path holds whitespace, with a leading +# slash so the index route ("/") is not empty. +paths="$(printf '%s' "$manifest" | + jq -r '"/", (.paths | keys[] | "/" + (split("/") | map(@uri) | join("/")))')" + +warm() { + local gateway="$1" path="$2" out rc err body code encoding size verdict=ok note= + err="$(mktemp)" + body="$(mktemp)" + # Asks for gzip as a browser does. Gateways drop the odd connection under + # parallel load, and answer 404 for an item they have not fetched yet, so + # retry with a pause: --fail makes a 404 an error that --retry-all-errors + # retries. + out="$(curl -sS -L --fail -o "$body" -H 'Accept-Encoding: gzip, deflate, br' \ + --retry 5 --retry-all-errors --retry-delay 3 --max-time 180 \ + -w '%{http_code} %header{content-encoding} %{size_download}' \ + "https://$gateway/$id$path" 2>"$err")" && rc=0 || rc=$? + read -r code encoding size <<<"$out" + # An absent header leaves `encoding` holding the size. + if [[ -z "${size:-}" ]]; then size="${encoding:-0}"; encoding=-; fi + # A 200 whose body broke off partway still fails: curl's status says so. + if [[ "$rc" -ne 0 || "${code:-000}" != 200 ]]; then + verdict=FAIL + [[ -s "$err" ]] && note="$(tail -1 "$err")" + elif [[ "$encoding" == - && "$(head -c 2 "$body" | od -An -tx1 | tr -d ' ')" == 1f8b ]]; then + verdict=FAIL + note='gzip bytes without Content-Encoding' + elif [[ "$encoding" == - && "$path" =~ \.(js|mjs|wasm)$ ]]; then + verdict=plain + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$verdict" "$gateway" "${code:-000}" \ + "$encoding" "$size" "$path" "$note" + rm -f "$err" "$body" +} +export -f warm +export id + +results="$(for gateway in $GATEWAYS; do + printf '%s\n' "$paths" | sed "s|^|$gateway |" +done | xargs -n 2 -P "$JOBS" bash -c 'warm "$1" "$2"' _)" + +failures="$(printf '%s\n' "$results" | awk -F'\t' '$1 == "FAIL"')" +if [[ -n "$failures" ]]; then + echo "verdict gateway status encoding bytes path note" + printf '%s\n' "$failures" +fi + +printf '%s\n' "$results" | awk -F'\t' ' + { n[$2]++; bytes[$2] += $5; if ($1 == "FAIL") bad[$2]++; if ($1 == "plain") plain[$2]++ } + END { + for (g in n) + printf "%s: %d paths, %d failed, %d JavaScript/wasm sent uncompressed, %.1f MB transferred\n", + g, n[g], bad[g] + 0, plain[g] + 0, bytes[g] / 1e6 + }' + +[[ -z "$failures" ]]