diff --git a/MODS.md b/MODS.md index cbd94c5..451bb2a 100644 --- a/MODS.md +++ b/MODS.md @@ -110,7 +110,9 @@ mod refers to exists (and that it depends on the mods those come from), and type `common/mod_api/`, which scripts import as `bworld/common`, `bworld/server`, `bworld/client`, `bworld/worldgen` and, with engine access, `bworld/engine`. Folders in `mods/` starting with `_` or `.` are ignored. -`deno task build` builds every mod in `mods/` and fails if any has errors; `deno task server` then loads them. +`deno task build` packs every mod in `mods/` into a `.bmod` in `server_mods/` and fails if any has errors; +`deno task server` then loads every `.bmod` there. `deno task pack-mod mods/copper_tools` packs one mod into a file to +share, see [Mod files](#mod-files). ## Mod layout @@ -118,6 +120,7 @@ with engine access, `bworld/engine`. Folders in `mods/` starting with `_` or `.` mods/ copper_tools/ manifest.json + deno.json # optional: the mod is its own deno project, see Mod files blocks/*.json models/*.json items/*.json @@ -201,8 +204,7 @@ program**, so the server and each client can number blocks differently. Saves an - Every `.png` in `textures/` becomes a texture with id `:`. Subfolders are joined with `_`, so `textures/ores/tin.png` becomes `:ores_tin`. -- Textures are 16×16. The server's build puts every installed mod's textures into one atlas with the base game's, and - clients download that atlas instead of using their own. +- Textures are 16×16. They go in the mod's `.bmod`, and clients put every mod's textures into one atlas when they join. - A missing texture shows the magenta and black checker (`engine:missing`) and logs a warning. It isn't a load error. - The engine's own textures use the `engine` namespace: `engine:missing` and the breaking cracks `engine:break_0` to `engine:break_8`. They come from `assets/`, not from a mod. @@ -1228,24 +1230,46 @@ function hook(target: T, method: K, handlers: { ## Delivery to clients -### Build +### Mod files -`deno task build` builds each mod in `mods/` into two places: +A mod is shipped as a `.bmod` file: the mod, built, in one zip. Servers load every `.bmod` in their `server_mods/` +folder, so installing a mod is dropping its file there and restarting. -``` -build/mods/// # public, served to players - manifest.json - data.json # all blocks, items, models, recipes, overrides and ores merged - common.js - client.js - worldgen.js -server_mods/// # private, outside the static root, never served - server.js +```sh +deno task pack-mod mods/copper_tools # writes copper_tools.bmod +deno task build # packs every mod in mods/ into server_mods/.bmod ``` -`` is a content hash of the mod's public files, so URLs never change content. The server sends them with -`Cache-Control: immutable`, and players only download a mod again when it changes. Textures from every mod go into the -server's atlas, which gets a hash-named URL the same way. +``` +copper_tools.bmod + manifest.json # the mod's manifest, with scripts and credits pointing into the zip + data.json # every block, model, item, recipe and ore, checked when it was packed + scripts/server.js # each script bundled into one module + scripts/client.js + scripts/worldgen.js + textures/.png # the texture copper_tools: + credits.md +``` + +- A mod is a Deno project. When its folder has a `deno.json`, its scripts are bundled with it, so its own imports and + npm or JSR packages work and end up in the bundle. Mods without one use the `deno.json` of the folder the build runs + in, like the ones in this repository. +- Packing checks the mod first, like `check-mods`, and refuses to pack one with errors. +- Packing the same mod twice gives the same bytes, so the file's hash only changes when the mod does. +- `deno task build` leaves other `.bmod` files in `server_mods/` alone, so mods from elsewhere can sit next to the ones + it builds. + +When the server starts it opens every `.bmod`, checks it again (it may come from anyone): its manifest and data, its +textures, the ids it uses from other mods, and that its dependencies are there. A problem in any mod stops the server +from starting, saying which mod and what's wrong, so the game never runs with some mods missing. Then it sorts them so +dependencies load first. + +Players get each mod as a `.bmod` too, **without its server script**: the server makes a copy without +`scripts/server.js` and serves it at `/mods/.bmod`, named by its hash so it can be cached forever. The code is in +`common/bmod.ts` (reading and writing), `tools/pack_mod.ts` and `server/load_bmods.ts`. + +The engine's own files (the client, its UI sprites and font, and the `engine:` textures) aren't in any mod. They're in +`build/`, which the server serves as it is. ### Joining @@ -1253,15 +1277,14 @@ server's atlas, which gets a hash-named URL the same way. client server │ hello { name, protocol } │ ├────────────────────────────────────────►│ a different protocol gets rejected { reason } - │ welcome { protocol, seed, atlas, │ mods[i] = { id, name, version, hash, - │ mods[] } │ data, client?, worldgen?, sha256 } - │◄────────────────────────────────────────┤ atlas = { png, json, sha256 } (paths on the server) + │ welcome { protocol, seed, mods[] } │ mods[i] = { id, name, version, sha256, file, size } + │◄────────────────────────────────────────┤ file is the .bmod's path on the server │ │ │ [cross-origin: confirm screen] │ - │ download the atlas and every mod file, │ - │ check each one's sha256, load content │ - │ in the listed order, run client │ - │ setup(), start chunk workers │ + │ download every .bmod, check each one's │ + │ sha256, build the texture atlas, load │ + │ content in the listed order, run │ + │ client setup(), start chunk workers │ │ │ │ ready { registry_hash } │ a different hash gets rejected { reason } ├────────────────────────────────────────►│ @@ -1271,17 +1294,20 @@ client server ``` - The client registers each mod's data **in the order the server lists them**. -- Every file is checked against its SHA-256 before it's used, and scripts are imported from the checked bytes (as +- Every `.bmod` is checked against its SHA-256 before it's opened, and scripts are imported from the checked bytes (as `blob:` URLs), so what runs is exactly what was checked. The client code is in `client/handshake.ts` and `client/mods.ts`. +- The client builds the texture atlas itself, from the engine's textures and the ones in the `.bmod` files + (`client/atlas.ts`). The server never needs to decode an image. - If any download, hash check or `setup` fails, the client disconnects and shows which mod failed. It never joins with some mods missing. - The `protocol` in `hello` is the game's protocol version (`PROTOCOL_VERSION` in `common/protocol.ts`). A mismatch is rejected before anything is downloaded. - Until `ready`, the connection isn't a player: other players don't see it, and anything it sends besides `ready` is ignored. A client that doesn't send `ready` within 60 seconds is rejected. -- Mod files and the atlas are served with `Access-Control-Allow-Origin: *` and cached for a year, since their paths - change whenever their content does. +- `.bmod` files are served with `Access-Control-Allow-Origin: *` and cached for a year, since their paths change + whenever their content does. The engine's assets are served with `Access-Control-Allow-Origin: *` too, for pages + from other origins. - Leaving a server reloads the page, so one server's mod code never stays loaded while playing on another. ## Security @@ -1294,7 +1320,7 @@ where the page came from: - **Page from one origin, game server on another** (a different server typed in the title screen, or `?server=`): mod code from that server runs with _the page's_ origin, including its local storage. The client shows the server's mod list and asks before loading anything, and remembers the answer per server and mod hash. The server needs CORS - headers on `build/mods/`. + headers on its `.bmod` downloads and assets, which it sends. - The hash check confirms the files are the ones the server listed. It doesn't protect against a malicious server. **Engine access.** A server mod with engine access controls the game server worker completely, but the worker still @@ -1307,6 +1333,9 @@ lists which mods use engine access. `--unstable-worker-options`). They can't read files, open connections or run programs. Everything they need goes through the mod API. Installing a server mod still means trusting it with the game world and everything players send. +**Server scripts stay on the server.** Players get each mod's `.bmod` without its server script, so a mod can keep +anti-cheat logic or anything else private there. + **Players.** Clients are untrusted. The server checks reach and the item being held for every break, place and interact, applies container rules on the server, validates form responses, and rate-limits mod channels. Server script code and tile data are never sent to clients. @@ -1474,8 +1503,8 @@ Worlds saved before any of this must load afterwards with nothing changed: ## Implementation plan -**Done so far:** the mod loader, build and delivery to clients (hashed downloads, the confirm screen for other servers, -credits); the data layer (blocks, block models with variants, items, recipes, ores); server scripts with components, +**Done so far:** the mod loader, `.bmod` mod files (packing, loading them from `server_mods/`, sending players a copy +without the server script), delivery to clients (hashed downloads, the confirm screen for other servers, credits); the data layer (blocks, block models with variants, items, recipes, ores); server scripts with components, events, commands, timers, storage, recipes, containers and container screens; worldgen features and ores; the base game's block behavior as a mod; world time with day and night. The code is in `common/mod_loader.ts`, `common/mod_data.ts`, `server/game/mod_runtime.ts`, `client/mods.ts` and `common/worldgen_loader.ts`. diff --git a/build.ts b/build.ts index 3603f70..15695dc 100644 --- a/build.ts +++ b/build.ts @@ -1,20 +1,16 @@ import { copy } from "@std/fs"; import { createCanvas, loadImage } from "@gfx/canvas-wasm"; import { ENGINE_TEXTURE_DIR, load_and_check, load_order, LoadedMod } from "./tools/check_mods.ts"; -import type { AtlasListing, ModData, ModListing } from "./common/mod_loader.ts"; +import { pack_mod } from "./tools/pack_mod.ts"; +import { BMOD_EXTENSION } from "./common/bmod.ts"; +import { ENGINE_TEXTURES_INDEX } from "./server/host.ts"; // all overridable so tests can build somewhere else const BUILD_FOLDER = Deno.env.get("BUILD_DIR") ?? "build"; const MODS_FOLDER = Deno.env.get("MODS_DIR") ?? "mods"; -// server scripts go here instead of the served build folder, players never get them +// where the server loads .bmod files from. the build puts every mod in mods/ there, next to any others const SERVER_MODS_FOLDER = Deno.env.get("SERVER_MODS_DIR") ?? "server_mods"; -// what the server reads at startup, see server/main.ts -export interface ServerModIndex { - atlas: AtlasListing; - mods: { listing: ModListing; server?: string }[]; -} - function clear_folder(folder: string) { try { Deno.removeSync(folder, { recursive: true }); @@ -58,90 +54,36 @@ async function build_fonts() { await copy("assets/fonts/m6x11.fnt", `${BUILD_FOLDER}/assets/fonts/m6x11.fnt`); } -function next_power_of_two(value: number): number { - return Math.pow(2, Math.ceil(Math.log2(value))); -} - -const SPRITE_SIZE = 16; - -function calculate_atlas_size(count: number) { - const raw_sprites_per_side = Math.ceil(Math.sqrt(count)); - const raw_size = raw_sprites_per_side * SPRITE_SIZE; - const size = next_power_of_two(raw_size); - - return { - sprites_per_side: size / SPRITE_SIZE, - size, - }; -} - -// one atlas with the engine's textures (engine:) and every mod's (:). -// named by its hash, so clients can cache it forever and download it from other origins -async function build_atlas(mods: LoadedMod[]): Promise { - const textures = new Map(); - for (const entry of Deno.readDirSync(ENGINE_TEXTURE_DIR)) { - if (entry.isFile && entry.name.endsWith(".png")) { - textures.set(`engine:${entry.name.replace(".png", "")}`, `${ENGINE_TEXTURE_DIR}/${entry.name}`); - } - } - for (const mod of mods) { - for (const [id, path] of mod.texture_files) { - textures.set(id, path); - } - } - - // + 1 for the missing texture - const atlas = calculate_atlas_size(textures.size + 1); - - const canvas = createCanvas(atlas.size, atlas.size); - const ctx = canvas.getContext("2d"); - - const atlas_info: Record = {}; - - // purple and black missing texture at x:0 y:0 wow ! - ctx.fillStyle = "magenta"; - ctx.fillRect(0, 0, 8, 8); - ctx.fillRect(8, 8, 8, 8); - ctx.fillStyle = "black"; - ctx.fillRect(8, 0, 8, 8); - ctx.fillRect(0, 8, 8, 8); - atlas_info["engine:missing"] = { x: 0, y: 0 }; - - let index = 1; - for (const [id, path] of [...textures].sort(([a], [b]) => a.localeCompare(b))) { - const sprite = await loadImage(path); - const row = Math.floor(index / atlas.sprites_per_side); - const column = index % atlas.sprites_per_side; - ctx.drawImage(sprite, column * SPRITE_SIZE, row * SPRITE_SIZE); - index += 1; - atlas_info[id] = { x: column, y: row }; - } - - const png = new Uint8Array(canvas.toBuffer()); - const json = new TextEncoder().encode(JSON.stringify(atlas_info)); - const hashes = { png: await sha256(png), json: await sha256(json) }; - const name = `assets/sprites/textures.${(hashes.png + hashes.json).slice(0, 12)}`; - Deno.writeFileSync(`${BUILD_FOLDER}/${name}.png`, png); - Deno.writeFileSync(`${BUILD_FOLDER}/${name}.json`, json); - return { png: `${name}.png`, json: `${name}.json`, sha256: hashes }; -} - -async function build_sprites(mods: LoadedMod[]) { +async function build_sprites() { for (const entry of Deno.readDirSync("assets/sprites")) { if (entry.name.endsWith(".png")) { await copy(`assets/sprites/${entry.name}`, `${BUILD_FOLDER}/assets/sprites/${entry.name}`); } } - return await build_atlas(mods); } -async function build_assets(mods: LoadedMod[]): Promise { +// the engine's textures (engine:), with a list of them. clients put them in the texture atlas with the mods' +function build_engine_textures() { + const folder = `${BUILD_FOLDER}/assets/textures`; + Deno.mkdirSync(folder, { recursive: true }); + const names: string[] = []; + for (const entry of [...Deno.readDirSync(ENGINE_TEXTURE_DIR)].sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.isFile && entry.name.endsWith(".png")) { + Deno.copyFileSync(`${ENGINE_TEXTURE_DIR}/${entry.name}`, `${folder}/${entry.name}`); + names.push(entry.name.replace(/\.png$/, "")); + } + } + Deno.writeTextFileSync(`${BUILD_FOLDER}/${ENGINE_TEXTURES_INDEX}`, JSON.stringify(names)); +} + +async function build_assets() { Deno.mkdirSync(`${BUILD_FOLDER}/assets`, { recursive: true }); await copy("assets/ASSETS.md", `${BUILD_FOLDER}/assets/ASSETS.md`); await build_fonts(); Deno.mkdirSync(`${BUILD_FOLDER}/assets/sprites`, { recursive: true }); - return await build_sprites(mods); + await build_sprites(); + build_engine_textures(); } // every mod in mods/, checked and sorted so dependencies load first. a broken mod fails the whole build, @@ -155,98 +97,25 @@ function load_mods(): LoadedMod[] { return load_order(mods); } -async function bundle_script(path: string, platform: "browser" | "deno"): Promise { - const result = await Deno.bundle({ entrypoints: [path], platform, write: false, minify: false }); - if (!result.success || !result.outputFiles?.length) { - throw new Error(`Couldn't bundle ${path}:\n${result.errors.map((e) => e.text).join("\n")}`); - } - return result.outputFiles[0].text(); -} - -async function sha256(bytes: Uint8Array | string) { - const data = typeof bytes === "string" ? new TextEncoder().encode(bytes) : bytes; - const digest = await crypto.subtle.digest("SHA-256", data); - return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); -} - -async function short_hash(parts: string[]) { - const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(parts.join("\0"))); - return [...new Uint8Array(digest)].slice(0, 6).map((b) => b.toString(16).padStart(2, "0")).join(""); -} - -// build/mods/// gets what players download, server_mods/ the server scripts -async function build_mods(mods: LoadedMod[], atlas: AtlasListing) { - clear_folder(SERVER_MODS_FOLDER); - const index: ServerModIndex = { atlas, mods: [] }; - +// every mod in mods/ into server_mods/.bmod. other .bmod files there are left alone, they're other people's mods +async function build_mods(mods: LoadedMod[]) { + Deno.mkdirSync(SERVER_MODS_FOLDER, { recursive: true }); + remove_old_format(); for (const mod of mods) { - const manifest = mod.manifest as { - name: string; - version: string; - scripts?: Record; - credits?: string; - }; - const scripts = manifest.scripts ?? {}; - - const data: ModData = { - blocks: mod.blocks.map((b) => b.json), - models: mod.models.map((m) => m.json), - items: mod.items.map((i) => i.json), - recipes: mod.recipes.map((r) => r.json), - ores: mod.ores.map((o) => o.json), - }; - const data_json = JSON.stringify(data); - const client = scripts.client ? await bundle_script(`${mod.dir}/${scripts.client}`, "browser") : undefined; - const worldgen = scripts.worldgen - ? await bundle_script(`${mod.dir}/${scripts.worldgen}`, "browser") - : undefined; - const server = scripts.server ? await bundle_script(`${mod.dir}/${scripts.server}`, "deno") : undefined; - - const credits = manifest.credits ? Deno.readTextFileSync(`${mod.dir}/${manifest.credits}`) : undefined; - - const hash = await short_hash([data_json, client ?? "", worldgen ?? "", credits ?? ""]); - const public_dir = `mods/${mod.id}/${hash}`; - Deno.mkdirSync(`${BUILD_FOLDER}/${public_dir}`, { recursive: true }); - - const listing: ModListing = { - id: mod.id, - name: manifest.name, - version: manifest.version, - hash, - data: `${public_dir}/data.json`, - sha256: { data: await sha256(data_json) }, - }; - Deno.writeTextFileSync(`${BUILD_FOLDER}/${listing.data}`, data_json); - if (client) { - listing.client = `${public_dir}/client.js`; - listing.sha256.client = await sha256(client); - Deno.writeTextFileSync(`${BUILD_FOLDER}/${listing.client}`, client); - } - if (worldgen) { - listing.worldgen = `${public_dir}/worldgen.js`; - listing.sha256.worldgen = await sha256(worldgen); - Deno.writeTextFileSync(`${BUILD_FOLDER}/${listing.worldgen}`, worldgen); - } - if (credits !== undefined) { - listing.credits = `${public_dir}/credits.md`; - listing.sha256.credits = await sha256(credits); - Deno.writeTextFileSync(`${BUILD_FOLDER}/${listing.credits}`, credits); - } - - const entry: ServerModIndex["mods"][number] = { listing }; - if (server) { - const server_dir = `${SERVER_MODS_FOLDER}/${mod.id}/${await short_hash([server])}`; - Deno.mkdirSync(server_dir, { recursive: true }); - entry.server = `${server_dir}/server.js`; - Deno.writeTextFileSync(entry.server, server); - } - index.mods.push(entry); + Deno.writeFileSync(`${SERVER_MODS_FOLDER}/${mod.id}${BMOD_EXTENSION}`, await pack_mod(mod)); } - - Deno.writeTextFileSync(`${SERVER_MODS_FOLDER}/index.json`, JSON.stringify(index, null, "\t")); console.log(`Mods: ${mods.map((m) => m.id).join(", ") || "none"}`); } +// builds before .bmod files put each mod's server script in a folder here with an index.json +function remove_old_format() { + for (const entry of Deno.readDirSync(SERVER_MODS_FOLDER)) { + if (entry.isDirectory || entry.name === "index.json") { + Deno.removeSync(`${SERVER_MODS_FOLDER}/${entry.name}`, { recursive: true }); + } + } +} + async function build_client() { const _result = await Deno.bundle({ entrypoints: ["./client/main.ts", "./client/workers/chunk_worker.ts"], @@ -262,16 +131,13 @@ async function build() { try { const now = performance.now(); clear_folder(BUILD_FOLDER); + // checked first: a mod with errors stops the build before any .bmod is written const mods = load_mods(); - const atlas = await build_assets(mods); + await build_assets(); await build_client(); - await build_mods(mods, atlas); + await build_mods(mods); console.log(`Built in ${(performance.now() - now).toFixed(2)}ms`); } catch (e) { - // no index means the server refuses to start, instead of running without some mods - try { - Deno.removeSync(`${SERVER_MODS_FOLDER}/index.json`); - } catch { /* wasn't there */ } console.log(e instanceof Error ? e.message : e); return false; } diff --git a/client/atlas.ts b/client/atlas.ts new file mode 100644 index 0000000..5afa482 --- /dev/null +++ b/client/atlas.ts @@ -0,0 +1,64 @@ +// the texture atlas: the engine's textures and every mod's in one image, built by the client when it joins a server +// from the textures in the server's .bmod files +import { TEXTURE_SIZE } from "$/common/constants.ts"; +import type { SpriteRegion } from "$/common/constants.ts"; + +export interface AtlasLayout { + // in pixels, a power of two + size: number; + // in sprites, by texture id. engine:missing is always at 0, 0 + regions: Record; +} + +// engine:missing first, then every other texture sorted by id, row by row +export function atlas_layout(ids: Iterable): AtlasLayout { + const sorted = [...new Set(ids)].filter((id) => id !== "engine:missing").sort((a, b) => a.localeCompare(b)); + const count = sorted.length + 1; + const size = 2 ** Math.ceil(Math.log2(Math.ceil(Math.sqrt(count)) * TEXTURE_SIZE)); + const per_row = size / TEXTURE_SIZE; + const regions: Record = { "engine:missing": { x: 0, y: 0 } }; + sorted.forEach((id, i) => { + regions[id] = { x: (i + 1) % per_row, y: Math.floor((i + 1) / per_row) }; + }); + return { size, regions }; +} + +// textures are png bytes by id. ones that don't decode show as missing +export async function build_atlas(textures: Map) { + const { size, regions } = atlas_layout(textures.keys()); + const canvas = new OffscreenCanvas(size, size); + const ctx = canvas.getContext("2d")!; + + // magenta and black checker + ctx.fillStyle = "magenta"; + ctx.fillRect(0, 0, 8, 8); + ctx.fillRect(8, 8, 8, 8); + ctx.fillStyle = "black"; + ctx.fillRect(8, 0, 8, 8); + ctx.fillRect(0, 8, 8, 8); + + await Promise.all([...textures].map(async ([id, png]) => { + const region = regions[id]; + try { + const image = await createImageBitmap(new Blob([png as Uint8Array], { type: "image/png" })); + ctx.drawImage(image, region.x * TEXTURE_SIZE, region.y * TEXTURE_SIZE); + image.close(); + } catch { + console.warn(`Texture ${id} isn't a png that can be shown, it will show as missing`); + regions[id] = regions["engine:missing"]; + } + })); + + return { image: canvas.transferToImageBitmap(), regions }; +} + +// the engine's own textures, which come with the client instead of a mod. see build_engine_textures in build.ts +export async function engine_textures(): Promise> { + const names: string[] = await (await fetch("/assets/textures/index.json")).json(); + const textures = new Map(); + await Promise.all(names.map(async (name) => { + const response = await fetch(`/assets/textures/${name}.png`); + textures.set(`engine:${name}`, new Uint8Array(await response.arrayBuffer())); + })); + return textures; +} diff --git a/client/handshake.ts b/client/handshake.ts index a58112b..484bf94 100644 --- a/client/handshake.ts +++ b/client/handshake.ts @@ -1,6 +1,5 @@ // connecting to a server and getting what it needs before joining, see "Delivery to clients" in MODS.md: // hello -> welcome, download and check everything, ready -> join -import type { AtlasListing } from "$/common/mod_loader.ts"; import { ClientMessage, PROTOCOL_VERSION, ServerMessage } from "$/common/protocol.ts"; const CONNECT_TIMEOUT_MS = 5000; @@ -189,17 +188,6 @@ export function code_url(bytes: Uint8Array): string { return URL.createObjectURL(new Blob([bytes], { type: "text/javascript" })); } -export async function load_atlas(address: ServerAddress, atlas: AtlasListing) { - const [png, json] = await Promise.all([ - fetch_verified(new URL(atlas.png, address.base), atlas.sha256.png, "the texture atlas"), - fetch_verified(new URL(atlas.json, address.base), atlas.sha256.json, "the texture atlas"), - ]); - return { - image: await createImageBitmap(new Blob([png], { type: "image/png" })), - regions: JSON.parse(new TextDecoder().decode(json)) as Record, - }; -} - // players ok a cross-origin server's mods once, per server and exact mod versions function trust_key(address: ServerAddress) { @@ -207,7 +195,7 @@ function trust_key(address: ServerAddress) { } function mod_versions(welcome: Welcome) { - return welcome.mods.map((mod) => `${mod.id}@${mod.hash}`).sort().join(","); + return welcome.mods.map((mod) => `${mod.id}@${mod.sha256}`).sort().join(","); } export function is_trusted(address: ServerAddress, welcome: Welcome): boolean { diff --git a/client/main.ts b/client/main.ts index 752aadc..90e297e 100644 --- a/client/main.ts +++ b/client/main.ts @@ -2,7 +2,8 @@ import { AssetManager } from "./assets.ts"; import { Client } from "./client.ts"; import { InputManager } from "./input_manager.ts"; import { Connection } from "./network.ts"; -import { connect, HandshakeError, is_trusted, join, load_atlas, remember_trust, ServerAddress } from "./handshake.ts"; +import { connect, HandshakeError, is_trusted, join, remember_trust, ServerAddress } from "./handshake.ts"; +import { build_atlas, engine_textures } from "./atlas.ts"; import { confirm_mods } from "./confirm_mods.ts"; import { ModLoadError } from "$/common/mod_loader.ts"; import { @@ -16,7 +17,7 @@ import { resize_canvas, } from "./renderer/mod.ts"; import { is_stopped, show_fatal_error } from "./fatal.ts"; -import { load_client_mods, set_mods_client } from "./mods.ts"; +import { download_mods, load_client_mods, set_mods_client } from "./mods.ts"; import type { GuiScreen } from "./gui/gui_screen.ts"; import { TitleScreen } from "./gui/title_screen.ts"; import { DisconnectedScreen } from "./gui/disconnected_screen.ts"; @@ -121,11 +122,16 @@ async function join_server(address: ServerAddress, name: string, status: (text: // textures, blocks and items all come from the server's mods, so this happens before anything else status("Downloading the server's mods..."); - const atlas = await load_atlas(address, welcome.atlas); + const bmods = await download_mods(welcome.mods, address.base); + const textures = await engine_textures(); + for (const bmod of bmods) { + for (const [id, png] of bmod.textures) textures.set(id, png); + } + const atlas = await build_atlas(textures); loaded_mods = true; AssetManager.instance.assets["bworld:textures"] = load_texture(atlas.image); AssetManager.instance.assets["bworld:textures_info"] = atlas.regions; - await load_client_mods(welcome.mods, address.base); + await load_client_mods(welcome.mods, bmods); console.log(`Mods: ${welcome.mods.map((mod) => `${mod.id} ${mod.version}`).join(", ") || "none"}`); status("Joining..."); diff --git a/client/mods.ts b/client/mods.ts index 7781077..4fdaf2c 100644 --- a/client/mods.ts +++ b/client/mods.ts @@ -1,9 +1,10 @@ -// loads the mods the server lists: registers their data, then runs their client scripts. +// loads the mods the server lists: downloads their .bmod files, registers their data, then runs their client scripts. // see "Delivery to clients" in MODS.md import { AIR } from "$/common/constants.ts"; import { BlockRegistry, EverythingRegistry } from "$/common/everything_registry.ts"; import type { ClientContext } from "$/common/mod_api/client.ts"; -import { ModData, ModListing, ModLoadError, register_mod_data } from "$/common/mod_loader.ts"; +import { ModListing, ModLoadError, register_mod_data } from "$/common/mod_loader.ts"; +import { type Bmod, read_bmod } from "$/common/bmod.ts"; import type { OreJson } from "$/common/mod_data.ts"; import { AIR_ID } from "$/common/protocol.ts"; import type { Client } from "./client.ts"; @@ -22,50 +23,50 @@ export function set_mods_client(game_client: Client) { client = game_client; } -// downloads every mod's files and checks them against the hashes the server listed, then registers the data and -// runs the client scripts. everything a mod runs is imported from the checked bytes, never fetched twice -export async function load_client_mods(listings: ModListing[], base: URL) { - const downloads = await Promise.all(listings.map(async (listing) => { - const get = async (path: string | undefined, sha256: string | undefined, what: string) => { - if (!path) return undefined; - try { - return await fetch_verified(new URL(path, base), sha256 ?? "", what); - } catch (e) { - throw new ModLoadError(listing.id, (e as Error).message); - } - }; - const [data, client, worldgen, credits] = await Promise.all([ - get(listing.data, listing.sha256.data, "its data"), - get(listing.client, listing.sha256.client, "its client script"), - get(listing.worldgen, listing.sha256.worldgen, "its worldgen script"), - get(listing.credits, listing.sha256.credits, "its credits"), - ]); - return { listing, data: JSON.parse(new TextDecoder().decode(data)) as ModData, client, worldgen, credits }; +// downloads every mod's .bmod and checks it against the hash the server listed. everything a mod runs comes from +// the checked bytes, never fetched twice +export async function download_mods(listings: ModListing[], base: URL): Promise { + return await Promise.all(listings.map(async (listing) => { + try { + const bytes = await fetch_verified(new URL(listing.file, base), listing.sha256, `${listing.name}`); + const bmod = read_bmod(bytes, `${listing.id}.bmod`); + if (bmod.manifest.id !== listing.id) throw new Error(`the server listed it as ${listing.id}`); + return bmod; + } catch (e) { + throw new ModLoadError(listing.id, (e as Error).message); + } })); +} - for (const { listing, credits } of downloads) { - if (credits) { - mod_credits.push({ name: listing.name, version: listing.version, text: new TextDecoder().decode(credits) }); +// registers the mods' data and runs their client scripts, in the order the server listed them +export async function load_client_mods(listings: ModListing[], bmods: Bmod[]) { + for (const [i, bmod] of bmods.entries()) { + if (bmod.credits) { + mod_credits.push({ name: listings[i].name, version: listings[i].version, text: bmod.credits }); } } - const recipes = register_mod_data(downloads.map(({ listing, data }) => ({ id: listing.id, data }))); + const recipes = register_mod_data(bmods.map((bmod) => ({ id: bmod.manifest.id, data: bmod.data }))); worldgen_mods.ores = recipes.ores; - worldgen_mods.scripts = downloads.flatMap(({ listing, worldgen }) => - worldgen ? [{ mod: listing.id, url: code_url(worldgen) }] : [] + worldgen_mods.scripts = bmods.flatMap((bmod) => + bmod.scripts.worldgen ? [{ mod: bmod.manifest.id, url: script_url(bmod.scripts.worldgen) }] : [] ); - for (const { listing, client } of downloads) { - if (!client) continue; - const module = await import(code_url(client)); + for (const [i, bmod] of bmods.entries()) { + if (!bmod.scripts.client) continue; + const module = await import(script_url(bmod.scripts.client)); if (typeof module.setup !== "function") { - throw new ModLoadError(listing.id, "the client script doesn't export a setup function"); + throw new ModLoadError(bmod.manifest.id, "the client script doesn't export a setup function"); } - await module.setup(client_context(listing)); + await module.setup(client_context(listings[i])); } } +function script_url(code: string) { + return code_url(new TextEncoder().encode(code)); +} + function client_context(listing: ModListing): ClientContext { const mod = listing.id; const not_yet = (name: string, where: string) => diff --git a/common/bmod.ts b/common/bmod.ts new file mode 100644 index 0000000..bff53a9 --- /dev/null +++ b/common/bmod.ts @@ -0,0 +1,201 @@ +// .bmod files: a built mod in one zip, what servers load from server_mods/ and send to players. see "Mod files" in +// MODS.md. the same code reads them on the server and in the browser, so it only works on bytes +import { unzipSync, zipSync } from "fflate"; +import type { ModelJson } from "./block_models.ts"; +import { + type BlockJson, + ID_PATTERN, + type ItemJson, + type ManifestJson, + type OreJson, + type RecipeJson, + validate_block, + validate_item, + validate_manifest, + validate_model, + validate_ore, + validate_recipe, +} from "./mod_data.ts"; +import type { ModData } from "./mod_loader.ts"; + +export const BMOD_EXTENSION = ".bmod"; + +// what a .bmod can hold at most unzipped, so a small file can't unzip into gigabytes +const MAX_FILES = 4096; +const MAX_UNZIPPED_BYTES = 64 * 1024 * 1024; +// written into every entry, so packing the same mod twice gives the same bytes and the same hash +const FIXED_TIME = new Date(Date.UTC(2020, 0, 1)); + +export type ScriptSide = "server" | "client" | "worldgen"; +export const SCRIPT_SIDES: ScriptSide[] = ["server", "client", "worldgen"]; + +export interface Bmod { + manifest: ManifestJson; + data: ModData; + // bundled javascript, one module per side + scripts: Partial>; + // png bytes by texture id, like copper_tools:copper_block + textures: Map; + credits?: string; +} + +export class BmodError extends Error { + constructor(file: string, problems: string[]) { + super(`${file}: ${problems.join("; ")}`); + this.name = "BmodError"; + } +} + +// the zip's layout: +// manifest.json the mod's manifest, with "scripts" and "credits" pointing into the zip +// data.json every block, model, item, recipe and ore +// scripts/.js +// textures/.png the texture : +// credits.md +export function write_bmod(bmod: Bmod): Uint8Array { + const text = (value: string) => new TextEncoder().encode(value); + const manifest: ManifestJson = { ...bmod.manifest, scripts: {} }; + delete manifest.credits; + + const files: Record = {}; + for (const side of SCRIPT_SIDES) { + const code = bmod.scripts[side]; + if (code !== undefined) { + files[`scripts/${side}.js`] = text(code); + manifest.scripts![side] = `scripts/${side}.js`; + } + } + if (Object.keys(manifest.scripts!).length === 0) delete manifest.scripts; + if (bmod.credits !== undefined) { + files["credits.md"] = text(bmod.credits); + manifest.credits = "credits.md"; + } + for (const [id, png] of bmod.textures) { + files[`textures/${id.split(":")[1]}.png`] = png; + } + files["manifest.json"] = text(JSON.stringify(manifest, null, "\t")); + files["data.json"] = text(JSON.stringify(bmod.data)); + + // sorted, so the order doesn't depend on how the files were collected + const sorted: Record = {}; + for (const name of Object.keys(files).sort()) sorted[name] = files[name]; + return zipSync(sorted, { mtime: FIXED_TIME, level: 9 }) as Uint8Array; +} + +// opens and checks a .bmod. file is only for error messages. throws BmodError listing what's wrong +export function read_bmod(bytes: Uint8Array, file: string): Bmod { + let files: Record; + let count = 0; + let total = 0; + try { + files = unzipSync(bytes, { + filter(entry) { + count += 1; + total += entry.originalSize; + if (count > MAX_FILES || total > MAX_UNZIPPED_BYTES) { + throw new Error(`unzips to more than ${MAX_FILES} files or ${MAX_UNZIPPED_BYTES >> 20} MB`); + } + return !entry.name.endsWith("/"); + }, + }); + } catch (e) { + throw new BmodError(file, [`isn't a readable .bmod: ${(e as Error).message}`]); + } + + const problems: string[] = []; + const json = (name: string): unknown => { + const content = files[name]; + if (!content) { + problems.push(`${name} is missing`); + return undefined; + } + try { + return JSON.parse(new TextDecoder().decode(content)); + } catch (e) { + problems.push(`${name} isn't valid json: ${(e as Error).message}`); + return undefined; + } + }; + const text = (name: string) => files[name] && new TextDecoder().decode(files[name]); + + const manifest = json("manifest.json") as ManifestJson | undefined; + const raw_data = json("data.json") as Partial | undefined; + if (!manifest || !raw_data) throw new BmodError(file, problems); + + for (const problem of validate_manifest(manifest, manifest.id)) problems.push(`manifest.json: ${problem}`); + if (problems.length > 0) throw new BmodError(file, problems); + const mod = manifest.id; + + const data: ModData = { + blocks: checked(raw_data.blocks, "blocks", validate_block, problems) as BlockJson[], + models: checked(raw_data.models, "models", validate_model, problems) as ModelJson[], + items: checked(raw_data.items, "items", validate_item, problems) as ItemJson[], + recipes: checked(raw_data.recipes, "recipes", validate_recipe, problems) as RecipeJson[], + ores: checked(raw_data.ores, "ores", validate_ore, problems) as OreJson[], + }; + // a mod only registers ids in its own namespace + for (const { id } of [...data.blocks, ...data.models, ...data.items]) { + if (typeof id === "string" && id.split(":")[0] !== mod) problems.push(`${id} isn't in this mod's namespace`); + } + + const scripts: Bmod["scripts"] = {}; + for (const [side, path] of Object.entries(manifest.scripts ?? {})) { + const code = text(path); + if (!SCRIPT_SIDES.includes(side as ScriptSide) || code === undefined) { + problems.push(`manifest.json: scripts.${side} ${path} isn't in the file`); + } else { + scripts[side as ScriptSide] = code; + } + } + const credits = manifest.credits === undefined ? undefined : text(manifest.credits); + if (manifest.credits !== undefined && credits === undefined) { + problems.push(`manifest.json: credits ${manifest.credits} isn't in the file`); + } + + const textures = new Map(); + for (const [name, content] of Object.entries(files)) { + const match = name.match(/^textures\/(.+)\.png$/); + if (!match) continue; + const id = `${mod}:${match[1]}`; + const size = png_size(content); + if (!ID_PATTERN.test(id)) { + problems.push(`${name}: texture names must be a-z, 0-9 and _`); + } else if (!size) { + problems.push(`${name}: isn't a png`); + } else if (size.width !== 16 || size.height !== 16) { + problems.push(`${name}: is ${size.width}×${size.height}, textures must be 16×16`); + } else { + textures.set(id, content); + } + } + + if (problems.length > 0) throw new BmodError(file, problems); + return { manifest, data, scripts, textures, credits }; +} + +// the same mod with its server script taken out, what players get +export function client_copy(bmod: Bmod): Bmod { + const scripts = { ...bmod.scripts }; + delete scripts.server; + return { ...bmod, scripts }; +} + +function checked(list: unknown, name: string, validate: (json: unknown) => string[], problems: string[]): unknown[] { + if (list === undefined) return []; + if (!Array.isArray(list)) { + problems.push(`data.json: ${name} must be a list`); + return []; + } + list.forEach((entry, i) => { + for (const problem of validate(entry)) problems.push(`data.json: ${name}[${i}]: ${problem}`); + }); + return list; +} + +// reads the size out of the png header instead of decoding the image +export function png_size(bytes: Uint8Array): { width: number; height: number } | undefined { + const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + if (bytes.length < 24 || !signature.every((b, i) => bytes[i] === b)) return undefined; + const view = new DataView(bytes.buffer, bytes.byteOffset); + return { width: view.getUint32(16), height: view.getUint32(20) }; +} diff --git a/common/mod_check.ts b/common/mod_check.ts new file mode 100644 index 0000000..e8fec0c --- /dev/null +++ b/common/mod_check.ts @@ -0,0 +1,157 @@ +// checks between mods: dependencies, load order, and ids one mod uses from another. pure, so the build, check-mods +// and the server (on .bmod files) all use it +import type { BlockJson, ItemJson, OreJson, RecipeJson } from "./mod_data.ts"; +import { BUILTIN_MODELS, type ModelJson } from "./block_models.ts"; + +export interface ModReport { + id: string; + errors: string[]; + warnings: string[]; +} + +export interface LoadedMod { + id: string; + dir: string; + report: ModReport; + manifest?: Record; + blocks: { file: string; json: BlockJson }[]; + models: { file: string; json: ModelJson }[]; + items: { file: string; json: ItemJson }[]; + recipes: { file: string; json: RecipeJson }[]; + ores: { file: string; json: OreJson }[]; + textures: string[]; + // absolute paths by texture id + texture_files: Map; +} + +// dependencies before the mods that need them, otherwise alphabetical +export function load_order(mods: LoadedMod[]): LoadedMod[] { + const by_id = new Map(mods.map((mod) => [mod.id, mod])); + const ordered: LoadedMod[] = []; + const state = new Map(); + + const visit = (mod: LoadedMod, path: string[]) => { + if (state.get(mod.id) === "done") return; + if (state.get(mod.id) === "visiting") { + throw new Error(`mods depend on each other in a circle: ${[...path, mod.id].join(" -> ")}`); + } + state.set(mod.id, "visiting"); + const dependencies = ((mod.manifest?.dependencies ?? []) as { id: string }[]).map((d) => d.id).sort(); + for (const dependency of dependencies) { + const other = by_id.get(dependency); + if (other) visit(other, [...path, mod.id]); + } + state.set(mod.id, "done"); + ordered.push(mod); + }; + + for (const mod of [...mods].sort((a, b) => a.id.localeCompare(b.id))) { + visit(mod, []); + } + return ordered; +} + +// engine_textures are the engine: texture ids, for telling whether a texture a mod uses exists +export function check_references(mods: LoadedMod[], engine_textures: Iterable) { + const block_owners = new Map(); + const item_owners = new Map(); + const model_owners = new Map(); + const add = (map: Map, id: string, mod: string) => map.set(id, [...(map.get(id) ?? []), mod]); + + const textures = new Set(engine_textures); + + for (const mod of mods) { + for (const { json } of mod.blocks) { + add(block_owners, json.id, mod.id); + if (json.item !== false) add(item_owners, json.id, mod.id); + } + for (const { json } of mod.items) add(item_owners, json.id, mod.id); + for (const { json } of mod.models) add(model_owners, json.id, mod.id); + for (const texture of mod.textures) textures.add(texture); + } + + const mod_ids = new Set(mods.map((mod) => mod.id)); + + for (const mod of mods) { + const { errors, warnings } = mod.report; + const dependencies = new Set( + ((mod.manifest?.dependencies ?? []) as { id: string }[]).map((dep) => dep.id), + ); + for (const dep of dependencies) { + if (!mod_ids.has(dep)) errors.push(`manifest.json: depends on "${dep}", which isn't installed`); + } + + const uses = (file: string, id: string, what: "block" | "item" | "texture" | "model") => { + const namespace = id.split(":")[0]; + if (namespace !== mod.id && namespace !== "engine" && !dependencies.has(namespace)) { + warnings.push(`${file}: uses ${id} but doesn't list "${namespace}" in dependencies`); + } + if (what === "model") { + if (!BUILTIN_MODELS[id] && !model_owners.has(id)) errors.push(`${file}: model ${id} doesn't exist`); + } else if (what === "texture") { + if (!textures.has(id)) warnings.push(`${file}: texture ${id} doesn't exist, it will show as missing`); + } else if (!(what === "block" ? block_owners : item_owners).has(id)) { + errors.push(`${file}: ${what} ${id} doesn't exist`); + } + }; + + for (const { file, json } of mod.blocks) { + if ((block_owners.get(json.id)?.length ?? 0) > 1) { + errors.push(`${file}: block ${json.id} is also defined by ${block_owners.get(json.id)!.join(", ")}`); + } + for (const variant of [json, ...Object.values(json.variants ?? {})]) { + if (variant.model) uses(file, variant.model, "model"); + const textures = variant.textures ?? {}; + for (const texture of typeof textures === "string" ? [textures] : Object.values(textures)) { + uses(file, texture, "texture"); + } + } + if (json.drops) uses(file, json.drops, "item"); + } + for (const { file, json } of mod.models) { + if ((model_owners.get(json.id)?.length ?? 0) > 1 || BUILTIN_MODELS[json.id]) { + errors.push(`${file}: model ${json.id} is defined more than once`); + } + for (const texture of Object.values(json.textures ?? {})) { + if (!texture.startsWith("#")) uses(file, texture, "texture"); + } + for (const element of json.elements) { + for (const face of Object.values(element.faces)) { + if (face && !face.texture.startsWith("#")) uses(file, face.texture, "texture"); + } + } + } + for (const { file, json } of mod.items) { + if ((item_owners.get(json.id)?.length ?? 0) > 1) { + errors.push(`${file}: item ${json.id} is also defined by ${item_owners.get(json.id)!.join(", ")}`); + } + uses(file, json.texture, "texture"); + if (json.places) uses(file, json.places, "block"); + } + for (const { file, json } of mod.recipes) { + switch (json.type) { + case "shaped": + for (const id of Object.values(json.key)) uses(file, id, "item"); + uses(file, json.result.id, "item"); + break; + case "furnace": + uses(file, json.input, "item"); + uses(file, json.output.id, "item"); + break; + case "fuel": + uses(file, json.item, "item"); + break; + case "smithing": + uses(file, json.tool, "item"); + uses(file, json.material.id, "item"); + if (json.addition) uses(file, json.addition, "item"); + uses(file, json.result, "item"); + break; + } + } + for (const { file, json } of mod.ores) { + uses(file, json.id, "block"); + uses(file, json.replaces, "block"); + } + } +} diff --git a/common/mod_loader.ts b/common/mod_loader.ts index f556966..8a51478 100644 --- a/common/mod_loader.ts +++ b/common/mod_loader.ts @@ -17,33 +17,22 @@ import type { ModelJson } from "./block_models.ts"; export interface ModData { blocks: BlockJson[]; // block models, see common/block_models.ts - models?: ModelJson[]; + models: ModelJson[]; items: ItemJson[]; recipes: RecipeJson[]; ores: OreJson[]; } -// a mod as the server lists it to clients, in load order. paths are relative to the server's root +// a mod as the server lists it to clients, in load order: the .bmod players download, without its server script export interface ModListing { id: string; name: string; version: string; - // short hash of everything public, the folder it's served from - hash: string; - data: string; - client?: string; - worldgen?: string; - // the manifest's credits file, markdown, shown on the credits screen - credits?: string; - // sha-256 in hex of each file, clients check these before using them - sha256: { data: string; client?: string; worldgen?: string; credits?: string }; -} - -// the texture atlas with every mod's textures, built by the server -export interface AtlasListing { - png: string; - json: string; - sha256: { png: string; json: string }; + // sha-256 in hex of the .bmod, clients check it before using anything in it + sha256: string; + // where it's served from, relative to the server's root. named by its hash, so it never changes + file: string; + size: number; } export class RecipeBook { @@ -70,7 +59,7 @@ export function register_mod_data(mods: { id: string; data: ModData }[]): Recipe throw new ModLoadError(id, message); }; try { - for (const model of data.models ?? []) { + for (const model of data.models) { EverythingRegistry.register("models", model.id, model); } for (const json of data.blocks) { diff --git a/common/protocol.ts b/common/protocol.ts index 648abbd..947414f 100644 --- a/common/protocol.ts +++ b/common/protocol.ts @@ -1,13 +1,13 @@ // messages sent between the client and the server, as json over a websocket import type { Faces } from "./constants.ts"; import type { ItemData } from "./inventory.ts"; -import type { AtlasListing, ModListing } from "./mod_loader.ts"; +import type { ModListing } from "./mod_loader.ts"; export const AIR_ID = "bworld:air"; // bump when a client and server of different versions can't play together. // the server rejects a different version before the client downloads anything -export const PROTOCOL_VERSION = 3; +export const PROTOCOL_VERSION = 4; export interface PlayerInfo { id: string; @@ -83,8 +83,7 @@ export type ServerMessage = type: "welcome"; protocol: number; seed: string; - atlas: AtlasListing; - // in load order, the client loads them before joining the world + // in load order, the client downloads and loads them before joining the world mods: ModListing[]; } // the connection is closed right after diff --git a/deno.json b/deno.json index 8070a7f..72bddb2 100644 --- a/deno.json +++ b/deno.json @@ -5,6 +5,7 @@ "server": "deno run --unstable-worker-options --allow-net --allow-read --allow-write --allow-env server/main.ts", "new-mod": "deno run --allow-read --allow-write tools/new_mod.ts", "check-mods": "deno run --allow-read --allow-run tools/check_mods.ts", + "pack-mod": "deno run --allow-read --allow-write --allow-run --allow-env tools/pack_mod.ts", "test": "deno test --allow-read --allow-write --allow-run tests/", "desktop": "deno run -A build.ts --once && deno desktop --allow-read --allow-write --allow-net --allow-env=BWORLD_SERVER,HOME,USERPROFILE,APPDATA,XDG_DATA_HOME --include build --include server_mods --include server/game/worker.ts --include desktop/config.json desktop/main.ts && deno run --allow-read --allow-write desktop/linux_launcher.ts dist/bworld" }, @@ -30,6 +31,7 @@ "@std/fs": "jsr:@std/fs@^1.0.23", "@std/http": "jsr:@std/http@^1.0.23", "@std/path": "jsr:@std/path@^1.0.0", + "fflate": "npm:fflate@^0.8.2", "gl-matrix": "npm:gl-matrix@^3.4.4", "marked": "npm:marked@^17.0.3" }, diff --git a/deno.lock b/deno.lock index 87c74e7..1482404 100644 --- a/deno.lock +++ b/deno.lock @@ -20,6 +20,7 @@ "jsr:@std/path@1": "1.1.4", "jsr:@std/path@^1.1.4": "1.1.4", "jsr:@std/streams@^1.0.17": "1.1.2", + "npm:fflate@~0.8.2": "0.8.3", "npm:gl-matrix@^3.4.4": "3.4.4", "npm:marked@^17.0.3": "17.0.3" }, @@ -95,6 +96,9 @@ } }, "npm": { + "fflate@0.8.3": { + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==" + }, "gl-matrix@3.4.4": { "integrity": "sha512-latSnyDNt/8zYUB6VIJ6PCh2jBjJX6gnDsoCZ7LyW7GkqrD51EWwa9qCoGixj8YqBtETQK/xY7OmpTF8xz1DdQ==" }, @@ -115,6 +119,7 @@ "jsr:@std/fs@^1.0.23", "jsr:@std/http@^1.0.23", "jsr:@std/path@1", + "npm:fflate@~0.8.2", "npm:gl-matrix@^3.4.4", "npm:marked@^17.0.3" ] diff --git a/desktop/main.ts b/desktop/main.ts index 7cb960d..606061a 100644 --- a/desktop/main.ts +++ b/desktop/main.ts @@ -6,7 +6,7 @@ import { fromFileUrl, join } from "@std/path"; import { type Host, serve_static, start_host } from "../server/host.ts"; -// where the project was built from: build/ and server_mods/ are bundled into the app with --include +// where the project was built from: build/ and server_mods/ (the .bmod files) are bundled into the app with --include const ROOT = fromFileUrl(new URL("../", import.meta.url)); const BUILD_DIR = join(ROOT, "build"); @@ -19,17 +19,16 @@ const default_server = Deno.env.get("BWORLD_SERVER") ?? config.server ?? ""; const win = new Deno.BrowserWindow({ title: "bworld", width: 1280, height: 720 }); // the singleplayer game server, started the first time singleplayer connects so players who only join other servers -// don't get a world made for them -let host: Host | undefined; +// don't get a world made for them. it plays the .bmod files the app was built with +let host: Promise | undefined; -function singleplayer_host(): Host { - if (!host) { +function singleplayer_host(): Promise { + host ??= (async () => { const worlds = join(data_dir(), "worlds"); Deno.mkdirSync(worlds, { recursive: true }); - host = start_host({ + return await start_host({ build_dir: BUILD_DIR, server_mods_dir: join(ROOT, "server_mods"), - root: ROOT, world_file: join(worlds, "world.json"), on_fatal(message) { console.error(message); @@ -37,7 +36,7 @@ function singleplayer_host(): Host { Deno.exit(1); }, }); - } + })(); return host; } @@ -48,7 +47,7 @@ win.addEventListener("close", async (event) => { } event.preventDefault(); try { - await host.shutdown(); + await (await host).shutdown(); } catch (e) { console.error((e as Error).message); } @@ -56,7 +55,7 @@ win.addEventListener("close", async (event) => { }); // deno desktop binds this to a private local address and opens the window on it, the port given here is ignored -Deno.serve((req) => { +Deno.serve(async (req) => { const url = new URL(req.url); if (url.pathname === "/") { // ?server= fills in the address field, without it the field would start with this app's own address. @@ -66,8 +65,16 @@ Deno.serve((req) => { start.searchParams.set("singleplayer", ""); return Response.redirect(start, 302); } - if (url.pathname === "/ws") { - return singleplayer_host().handle(req); + // the singleplayer server's socket and its .bmod downloads + if (url.pathname === "/ws" || url.pathname.startsWith("/mods/")) { + try { + return (await singleplayer_host()).handle(req); + } catch (e) { + // its mods have problems, the title screen shows why the connection failed + console.error((e as Error).message); + host = undefined; + return new Response((e as Error).message, { status: 500 }); + } } return serve_static(BUILD_DIR, req, url); }); diff --git a/server/game/game_server.ts b/server/game/game_server.ts index 2bb410e..047a4f3 100644 --- a/server/game/game_server.ts +++ b/server/game/game_server.ts @@ -24,7 +24,7 @@ import { PROTOCOL_VERSION, ServerMessage, } from "$/common/protocol.ts"; -import type { AtlasListing, ModListing, RecipeBook } from "$/common/mod_loader.ts"; +import type { ModListing, RecipeBook } from "$/common/mod_loader.ts"; import type { WorldgenSetup } from "$/common/generation.ts"; import { CYCLE_TICKS, format_clock, time_of_day, TIMES_OF_DAY } from "$/common/time.ts"; import { ModRuntime, run_guarded } from "./mod_runtime.ts"; @@ -91,7 +91,6 @@ interface SavedTile { // the loaded mods, see load_mods.ts export interface GameMods { listings: ModListing[]; - atlas: AtlasListing; recipes: RecipeBook; worldgen?: WorldgenSetup; runtime: ModRuntime; @@ -485,7 +484,6 @@ export class GameServer { type: "welcome", protocol: PROTOCOL_VERSION, seed: this.world.seed, - atlas: this.#mods.atlas, mods: this.#mods.listings, } satisfies ServerMessage, ), diff --git a/server/game/host_protocol.ts b/server/game/host_protocol.ts index 659ec12..a226496 100644 --- a/server/game/host_protocol.ts +++ b/server/game/host_protocol.ts @@ -1,4 +1,4 @@ -import type { AtlasListing, ModData, ModListing } from "$/common/mod_loader.ts"; +import type { ModData, ModListing } from "$/common/mod_loader.ts"; // messages between server/main.ts (the host) and the game server worker @@ -7,7 +7,6 @@ export type HostToGame = type: "init"; save: string | undefined; default_seed: string; - atlas: AtlasListing; // in load order, with the scripts' code since the worker can't read files mods: { listing: ModListing; data: ModData; server_code?: string; worldgen_code?: string }[]; } diff --git a/server/game/load_mods.ts b/server/game/load_mods.ts index af155b3..1cf9c60 100644 --- a/server/game/load_mods.ts +++ b/server/game/load_mods.ts @@ -1,7 +1,7 @@ // starts a game server with its mods: registers their data, then imports worldgen and server scripts. // the worker loads built mods, tests can load mods straight from their source folders import { register_mod_data } from "$/common/mod_loader.ts"; -import type { AtlasListing, ModData, ModListing } from "$/common/mod_loader.ts"; +import type { ModData, ModListing } from "$/common/mod_loader.ts"; import { load_worldgen } from "$/common/worldgen_loader.ts"; import { GameHost, GameServer } from "./game_server.ts"; import { ModRuntime } from "./mod_runtime.ts"; @@ -18,7 +18,6 @@ export async function start_game( host: GameHost, save: string | undefined, default_seed: string, - atlas: AtlasListing, mods: ServerModSource[], ): Promise { const recipes = register_mod_data(mods.map((mod) => ({ id: mod.listing.id, data: mod.data }))); @@ -31,7 +30,6 @@ export async function start_game( const runtime = new ModRuntime(); const game = new GameServer(host, save, default_seed, { listings: mods.map((mod) => mod.listing), - atlas, recipes, worldgen, runtime, diff --git a/server/game/worker.ts b/server/game/worker.ts index b7a70d5..c023926 100644 --- a/server/game/worker.ts +++ b/server/game/worker.ts @@ -54,7 +54,6 @@ async function init(message: Extract) { }, message.save, message.default_seed, - message.atlas, message.mods.map((mod) => ({ listing: mod.listing, data: mod.data, diff --git a/server/host.ts b/server/host.ts index 4ce9552..07585ba 100644 --- a/server/host.ts +++ b/server/host.ts @@ -1,20 +1,20 @@ // the part of a game server that has permissions: files and networking. the game itself runs in a worker with none. // server/main.ts runs it as a dedicated server, and the desktop app runs it for singleplayer import { serveDir } from "@std/http/file-server"; -import { isAbsolute, join } from "@std/path"; +import { join } from "@std/path"; import { GameToHost, HostToGame } from "./game/host_protocol.ts"; -import type { ServerModIndex } from "../build.ts"; +import { load_bmods } from "./load_bmods.ts"; const MAX_MESSAGE_SIZE = 4096; +// in build/, see build.ts +export const ENGINE_TEXTURES_INDEX = "assets/textures/index.json"; const SHUTDOWN_TIMEOUT_MS = 5000; export interface HostOptions { - // what deno task build made: the client, assets and mods players download + // what deno task build made: the client and the engine's assets build_dir: string; - // the server scripts, with index.json + // the .bmod files to load server_mods_dir: string; - // what the paths in server_mods/index.json are relative to, the folder the build ran in - root: string; world_file: string; seed?: string; // the game can't go on, like the worker crashing @@ -27,9 +27,12 @@ export interface Host { shutdown(): Promise; } -export function start_host(options: HostOptions): Host { - const mods = read_mods(options); - console.log(`Mods: ${mods.mods.map((mod) => `${mod.listing.id} ${mod.listing.version}`).join(", ") || "none"}`); +// throws when the mods have problems, saying what they are +export async function start_host(options: HostOptions): Promise { + const mods = await load_bmods(options.server_mods_dir, engine_texture_ids(options.build_dir)); + console.log(`Mods: ${mods.map((mod) => `${mod.listing.id} ${mod.listing.version}`).join(", ") || "none"}`); + // what players download, by path + const downloads = new Map(mods.map((mod) => [`/${mod.listing.file}`, mod.client_bytes])); const game = new Worker(new URL("./game/worker.ts", import.meta.url), { type: "module", @@ -78,7 +81,12 @@ export function start_host(options: HostOptions): Host { type: "init", save: read_world(options.world_file), default_seed: options.seed ?? crypto.randomUUID(), - ...mods, + mods: mods.map(({ listing, bmod }) => ({ + listing, + data: bmod.data, + server_code: bmod.scripts.server, + worldgen_code: bmod.scripts.worldgen, + })), }); function handle_socket(socket: WebSocket) { @@ -117,6 +125,17 @@ export function start_host(options: HostOptions): Host { if (url.pathname === "/") { return Response.redirect(new URL("/client/", url), 302); } + const download = downloads.get(url.pathname); + if (download) { + // named by its hash, so it never changes and pages on other origins can cache it forever + return new Response(download, { + headers: { + "content-type": "application/zip", + "access-control-allow-origin": "*", + "cache-control": "public, max-age=31536000, immutable", + }, + }); + } return serve_static(options.build_dir, req, url); }, @@ -155,40 +174,18 @@ function write_world(file: string, data: string) { Deno.renameSync(`${file}.tmp`, file); } -// what deno task build made. no index means the build failed or never ran, and the game never runs without its mods -function read_mods(options: HostOptions): Pick, "mods" | "atlas"> { - let index: ServerModIndex; - try { - index = JSON.parse(Deno.readTextFileSync(join(options.server_mods_dir, "index.json"))); - } catch { - throw new Error( - `No ${options.server_mods_dir}/index.json, run deno task build first (it also reports mod errors)`, - ); - } - const from_root = (path: string) => isAbsolute(path) ? path : join(options.root, path); - return { - atlas: index.atlas, - mods: index.mods.map(({ listing, server }) => ({ - listing, - data: JSON.parse(Deno.readTextFileSync(join(options.build_dir, listing.data))), - server_code: server ? Deno.readTextFileSync(from_root(server)) : undefined, - worldgen_code: listing.worldgen - ? Deno.readTextFileSync(join(options.build_dir, listing.worldgen)) - : undefined, - })), - }; +// the engine's textures, listed by the build next to the client's assets. engine:missing is drawn by code +export function engine_texture_ids(build_dir: string): string[] { + const names: string[] = JSON.parse(Deno.readTextFileSync(join(build_dir, ENGINE_TEXTURES_INDEX))); + return ["engine:missing", ...names.map((name) => `engine:${name}`)]; } -// mod files and the atlas are named by their hash, so they never change and pages on other origins can load them +// the client and the engine's assets. pages on other origins load the engine's textures from here too export async function serve_static(build_dir: string, req: Request, url: URL) { const response = await serveDir(req, { fsRoot: build_dir, quiet: true }); - const shared = url.pathname.startsWith("/mods/") || url.pathname.startsWith("/assets/"); - if (shared && response.ok) { + if (url.pathname.startsWith("/assets/") && response.ok) { const headers = new Headers(response.headers); headers.set("Access-Control-Allow-Origin", "*"); - if (url.pathname.startsWith("/mods/") || /^\/assets\/sprites\/textures\.[0-9a-f]+\./.test(url.pathname)) { - headers.set("Cache-Control", "public, max-age=31536000, immutable"); - } return new Response(response.body, { status: response.status, statusText: response.statusText, headers }); } return response; diff --git a/server/load_bmods.ts b/server/load_bmods.ts new file mode 100644 index 0000000..3adcb90 --- /dev/null +++ b/server/load_bmods.ts @@ -0,0 +1,111 @@ +// the server's mods: every .bmod in server_mods/, checked, sorted so dependencies load first, and with the copy +// players download (the same mod without its server script) ready to serve +import { join } from "@std/path"; +import { type Bmod, BMOD_EXTENSION, client_copy, read_bmod, write_bmod } from "$/common/bmod.ts"; +import { check_references, load_order, type LoadedMod } from "$/common/mod_check.ts"; +import type { ModListing } from "$/common/mod_loader.ts"; + +export interface ServerMod { + bmod: Bmod; + listing: ModListing; + // what's served at listing.file + client_bytes: Uint8Array; +} + +export class ModsError extends Error { + constructor(problems: string[]) { + super(`Mods have errors:\n${problems.map((p) => ` ${p}`).join("\n")}`); + this.name = "ModsError"; + } +} + +// engine_textures are the engine: texture ids, for checking the textures mods use. a mod with problems stops the +// whole server from starting, the game never runs with some mods missing +export async function load_bmods(dir: string, engine_textures: Iterable): Promise { + let files: string[]; + try { + files = [...Deno.readDirSync(dir)] + .filter((entry) => entry.isFile && entry.name.endsWith(BMOD_EXTENSION)) + .map((entry) => entry.name) + .sort(); + } catch (e) { + if (e instanceof Deno.errors.NotFound) return []; + throw e; + } + + const problems: string[] = []; + const by_id = new Map(); + for (const file of files) { + try { + const bmod = read_bmod(Deno.readFileSync(join(dir, file)), file); + const other = by_id.get(bmod.manifest.id); + if (other) { + problems.push(`${file} and ${other.file} are both the mod "${bmod.manifest.id}"`); + continue; + } + by_id.set(bmod.manifest.id, { file, bmod }); + } catch (e) { + problems.push((e as Error).message); + } + } + + // the same checks check-mods runs on mod folders + const loaded = [...by_id.values()].map(({ file, bmod }) => as_loaded_mod(file, bmod)); + check_references(loaded, engine_textures); + for (const mod of loaded) { + const dependencies = (mod.manifest?.dependencies ?? []) as { id: string }[]; + for (const { id } of dependencies) { + if (!by_id.has(id)) mod.report.errors.push(`depends on "${id}", which isn't in ${dir}`); + } + problems.push(...mod.report.errors.map((error) => `${mod.dir}: ${error}`)); + for (const warning of mod.report.warnings) console.warn(`${mod.dir}: ${warning}`); + } + let ordered: LoadedMod[] = []; + try { + ordered = load_order(loaded); + } catch (e) { + problems.push((e as Error).message); + } + if (problems.length > 0) throw new ModsError(problems); + + return await Promise.all(ordered.map(async (mod) => { + const bmod = by_id.get(mod.id)!.bmod; + const client_bytes = write_bmod(client_copy(bmod)); + const sha256 = await hex_sha256(client_bytes); + return { + bmod, + client_bytes, + listing: { + id: bmod.manifest.id, + name: bmod.manifest.name, + version: bmod.manifest.version, + sha256, + file: `mods/${sha256}${BMOD_EXTENSION}`, + size: client_bytes.length, + }, + }; + })); +} + +// in the shape the checks in common/mod_check.ts take +function as_loaded_mod(file: string, bmod: Bmod): LoadedMod { + const entries = (list: T[]) => list.map((json) => ({ file: `${file} data.json`, json })); + return { + id: bmod.manifest.id, + dir: file, + report: { id: bmod.manifest.id, errors: [], warnings: [] }, + manifest: bmod.manifest as unknown as Record, + blocks: entries(bmod.data.blocks), + models: entries(bmod.data.models), + items: entries(bmod.data.items), + recipes: entries(bmod.data.recipes), + ores: entries(bmod.data.ores), + textures: [...bmod.textures.keys()], + texture_files: new Map(), + }; +} + +async function hex_sha256(bytes: Uint8Array) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} diff --git a/server/main.ts b/server/main.ts index 92ff0d1..b67b7d3 100644 --- a/server/main.ts +++ b/server/main.ts @@ -5,10 +5,9 @@ const PORT = Number(Deno.env.get("PORT") ?? 8000); let host: Host; try { - host = start_host({ + host = await start_host({ build_dir: Deno.env.get("BUILD_DIR") ?? "build", server_mods_dir: Deno.env.get("SERVER_MODS_DIR") ?? "server_mods", - root: ".", world_file: Deno.env.get("WORLD_FILE") ?? "world.json", seed: Deno.env.get("SEED"), on_fatal(message) { diff --git a/tests/bmod_test.ts b/tests/bmod_test.ts new file mode 100644 index 0000000..0f98f8b --- /dev/null +++ b/tests/bmod_test.ts @@ -0,0 +1,216 @@ +// .bmod files: packing mods, reading them back, and the server loading a folder of them +import { assert, assertEquals, assertRejects, assertStringIncludes, assertThrows } from "@std/assert"; +import { zipSync } from "fflate"; +import { atlas_layout } from "$/client/atlas.ts"; +import { read_bmod, write_bmod } from "$/common/bmod.ts"; +import { load_and_check } from "$/tools/check_mods.ts"; +import { pack_mod } from "$/tools/pack_mod.ts"; +import { create_mod } from "$/tools/new_mod.ts"; +import { load_bmods } from "$/server/load_bmods.ts"; +import { start_host } from "$/server/host.ts"; +import { copy_dir } from "./helpers.ts"; + +const ENGINE_TEXTURES = ["engine:missing", ...Array.from({ length: 9 }, (_, i) => `engine:break_${i}`)]; + +// mods/bworld and one made from the template, packed into a folder of .bmod files +async function packed_mods() { + const sources = Deno.makeTempDirSync({ prefix: "bworld_bmod_src_" }); + copy_dir("mods/bworld", `${sources}/bworld`); + create_mod("copper_tools", "Copper Tools", sources); + const mods = load_and_check(sources); + for (const mod of mods) assertEquals(mod.report.errors, [], mod.id); + + const folder = Deno.makeTempDirSync({ prefix: "bworld_bmod_" }); + for (const mod of mods) Deno.writeFileSync(`${folder}/${mod.id}.bmod`, await pack_mod(mod)); + return { sources, folder, mods }; +} + +Deno.test("packing a mod keeps its data, scripts, textures and credits, and gives the same bytes every time", async () => { + const { sources, folder, mods } = await packed_mods(); + const template = mods.find((mod) => mod.id === "copper_tools")!; + const bytes = Deno.readFileSync(`${folder}/copper_tools.bmod`); + assertEquals(await pack_mod(template), bytes); + + const bmod = read_bmod(bytes, "copper_tools.bmod"); + assertEquals(bmod.manifest.id, "copper_tools"); + assertEquals(bmod.data.blocks.map((b) => b.id), template.blocks.map((b) => b.json.id)); + assertEquals(bmod.data.recipes, template.recipes.map((r) => r.json)); + assertEquals([...bmod.textures.keys()].sort(), [...template.textures].sort()); + assertEquals(Object.keys(bmod.scripts).sort(), ["client", "server", "worldgen"]); + assertStringIncludes(bmod.scripts.server!, "setup"); + assert(bmod.credits); + + const bworld = read_bmod(Deno.readFileSync(`${folder}/bworld.bmod`), "bworld.bmod"); + assertEquals(bworld.data.blocks.length, 21); + assertEquals(Object.keys(bworld.scripts), ["server"]); + + Deno.removeSync(sources, { recursive: true }); + Deno.removeSync(folder, { recursive: true }); +}); + +Deno.test("broken .bmod files are turned away, saying why", () => { + const text = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)); + const manifest = { format_version: 1, id: "broken", name: "Broken", version: "1.0.0" }; + const zip = (files: Record) => zipSync(files); + + assertThrows(() => read_bmod(new TextEncoder().encode("not a zip"), "a.bmod"), Error, "isn't a readable .bmod"); + assertThrows(() => read_bmod(zip({ "data.json": text({}) }), "b.bmod"), Error, "manifest.json is missing"); + assertThrows( + () => + read_bmod( + zip({ + "manifest.json": text(manifest), + "data.json": text({ blocks: [{ id: "other:thing", textures: "broken:x" }, { id: "broken:y" }] }), + }), + "c.bmod", + ), + Error, + "other:thing isn't in this mod's namespace", + ); + assertThrows( + () => + read_bmod( + zip({ + "manifest.json": text({ ...manifest, scripts: { server: "scripts/server.js" } }), + "data.json": text({}), + }), + "d.bmod", + ), + Error, + "scripts.server scripts/server.js isn't in the file", + ); + // a texture that isn't 16x16 + const ui = Deno.readFileSync("assets/sprites/ui.png"); + assertThrows( + () => + read_bmod( + zip({ "manifest.json": text(manifest), "data.json": text({}), "textures/big.png": ui }), + "e.bmod", + ), + Error, + "textures must be 16×16", + ); + // too much once unzipped, without unzipping it all + const huge = new Uint8Array(65 * 1024 * 1024); + assertThrows( + () => read_bmod(zip({ "manifest.json": text(manifest), "data.json": text({}), "big.bin": huge }), "f.bmod"), + Error, + "unzips to more than", + ); +}); + +Deno.test("the server loads every .bmod in its folder, dependencies first, and players get them without server scripts", async () => { + const { sources, folder } = await packed_mods(); + const mods = await load_bmods(folder, ENGINE_TEXTURES); + assertEquals(mods.map((mod) => mod.listing.id), ["bworld", "copper_tools"]); + + for (const mod of mods) { + assertEquals(mod.listing.file, `mods/${mod.listing.sha256}.bmod`); + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", mod.client_bytes)); + assertEquals([...digest].map((b) => b.toString(16).padStart(2, "0")).join(""), mod.listing.sha256); + const copy = read_bmod(mod.client_bytes, mod.listing.file); + assertEquals(copy.scripts.server, undefined, "server scripts never go to players"); + assertEquals(copy.data, mod.bmod.data); + } + assert(mods[1].bmod.scripts.server, "the server still has it"); + + // the same mod twice, and a mod whose dependency isn't there + Deno.copyFileSync(`${folder}/copper_tools.bmod`, `${folder}/copper_tools_again.bmod`); + await assertRejects(() => load_bmods(folder, ENGINE_TEXTURES), Error, 'are both the mod "copper_tools"'); + Deno.removeSync(`${folder}/copper_tools_again.bmod`); + Deno.removeSync(`${folder}/bworld.bmod`); + await assertRejects(() => load_bmods(folder, ENGINE_TEXTURES), Error, `depends on "bworld"`); + + Deno.removeSync(sources, { recursive: true }); + Deno.removeSync(folder, { recursive: true }); +}); + +Deno.test("the host serves each mod's player copy by its hash, and a game made from .bmod files plays", async () => { + const { sources, folder } = await packed_mods(); + const build = Deno.makeTempDirSync({ prefix: "bworld_build_" }); + Deno.mkdirSync(`${build}/assets/textures`, { recursive: true }); + Deno.writeTextFileSync( + `${build}/assets/textures/index.json`, + JSON.stringify(ENGINE_TEXTURES.slice(1).map((id) => id.split(":")[1])), + ); + const world = `${build}/world.json`; + + const host = await start_host({ + build_dir: build, + server_mods_dir: folder, + world_file: world, + seed: "bmod-seed", + on_fatal: (message) => { + throw new Error(message); + }, + }); + const listings = (await load_bmods(folder, ENGINE_TEXTURES)).map((mod) => mod.listing); + for (const listing of listings) { + const response = await host.handle(new Request(`http://localhost/${listing.file}`)); + assertEquals(response.status, 200); + assertEquals(response.headers.get("access-control-allow-origin"), "*"); + const bmod = read_bmod(new Uint8Array(await response.arrayBuffer()), listing.file); + assertEquals(bmod.manifest.id, listing.id); + } + assertEquals((await host.handle(new Request("http://localhost/mods/nope.bmod"))).status, 404); + + // the game worker loaded both mods, including the template's server script, and saves + await host.shutdown(); + const saved = JSON.parse(Deno.readTextFileSync(world)); + assertEquals(saved.seed, "bmod-seed"); + + for (const dir of [sources, folder, build]) Deno.removeSync(dir, { recursive: true }); +}); + +Deno.test("a mod with its own deno.json is bundled with it", async () => { + const sources = Deno.makeTempDirSync({ prefix: "bworld_bmod_project_" }); + copy_dir("mods/bworld", `${sources}/bworld`); + const mod = `${sources}/project`; + Deno.mkdirSync(`${mod}/scripts/lib`, { recursive: true }); + Deno.writeTextFileSync( + `${mod}/manifest.json`, + JSON.stringify({ + format_version: 1, + id: "project", + name: "Project", + version: "1.0.0", + scripts: { server: "scripts/server.ts" }, + }), + ); + // only the mod's own import map knows "lib/" + Deno.writeTextFileSync(`${mod}/deno.json`, JSON.stringify({ imports: { "lib/": "./scripts/lib/" } })); + Deno.writeTextFileSync(`${mod}/scripts/lib/greeting.ts`, `export const GREETING = "hello from a deno project";\n`); + Deno.writeTextFileSync( + `${mod}/scripts/server.ts`, + `import { GREETING } from "lib/greeting.ts";\n// deno-lint-ignore no-explicit-any\nexport function setup(ctx: any) {\n\tctx.log(GREETING);\n}\n`, + ); + + const project = load_and_check(sources).find((m) => m.id === "project")!; + assertEquals(project.report.errors, []); + const bmod = read_bmod(await pack_mod(project), "project.bmod"); + assertStringIncludes(bmod.scripts.server!, "hello from a deno project"); + Deno.removeSync(sources, { recursive: true }); +}); + +Deno.test("the atlas puts the missing texture first and the rest in order", () => { + const layout = atlas_layout(["b:two", "a:one", "engine:break_0", "a:one"]); + assertEquals(layout.size, 32); + assertEquals(layout.regions, { + "engine:missing": { x: 0, y: 0 }, + "a:one": { x: 1, y: 0 }, + "b:two": { x: 0, y: 1 }, + "engine:break_0": { x: 1, y: 1 }, + }); + assertEquals(atlas_layout(Array.from({ length: 100 }, (_, i) => `m:t${i}`)).size, 256); +}); + +// write_bmod without anything optional still makes a file read_bmod takes +Deno.test("a mod with only data packs and reads", () => { + const bmod = { + manifest: { format_version: 1, id: "tiny", name: "Tiny", version: "1.0.0" }, + data: { blocks: [], models: [], items: [], recipes: [], ores: [] }, + scripts: {}, + textures: new Map(), + }; + assertEquals(read_bmod(write_bmod(bmod), "tiny.bmod").manifest.id, "tiny"); +}); diff --git a/tests/handshake_test.ts b/tests/handshake_test.ts index 7299d3e..625b13c 100644 --- a/tests/handshake_test.ts +++ b/tests/handshake_test.ts @@ -30,7 +30,7 @@ Deno.test("welcome lists what to download, and nothing happens until ready", asy const [welcome] = take(2); assertEquals(welcome.type, "welcome"); assertEquals(welcome.mods.map((m: { id: string }) => m.id), ["bworld"]); - assert(welcome.atlas.png && welcome.atlas.sha256); + assert(welcome.mods.every((m: { sha256: string; file: string }) => m.sha256 && m.file)); assert(!("players" in welcome) && !("changes" in welcome), "welcome shouldn't have world state"); // not in the world yet: others don't hear about bob, and bob can't act diff --git a/tests/helpers.ts b/tests/helpers.ts index 44da9b4..f780d17 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -19,9 +19,9 @@ export function mod_sources(mods_dir: string): ServerModSource[] { id: mod.id, name: String(mod.manifest?.name), version: String(mod.manifest?.version), - hash: "source", - data: `mods/${mod.id}/source/data.json`, - sha256: { data: "" }, + sha256: "source", + file: `mods/${mod.id}.bmod`, + size: 0, }, data: { blocks: mod.blocks.map((b) => b.json), @@ -50,8 +50,7 @@ export async function test_game(mods_dir: string, save?: string, seed = "test-se closed.add(conn); }, }; - const atlas = { png: "atlas.png", json: "atlas.json", sha256: { png: "", json: "" } }; - const game: GameServer = await start_game(host, save, seed, atlas, mod_sources(mods_dir)); + const game: GameServer = await start_game(host, save, seed, mod_sources(mods_dir)); // deno-lint-ignore no-explicit-any const take = (conn: number): any[] => { const messages = outbox.get(conn) ?? []; diff --git a/tools/check_mods.ts b/tools/check_mods.ts index c010e62..24e53cc 100644 --- a/tools/check_mods.ts +++ b/tools/check_mods.ts @@ -1,11 +1,8 @@ // deno task check-mods // validates every mod in mods/: manifests, data files, textures, references between them, and typechecks scripts import { - BlockJson, FORMAT_VERSION, - ItemJson, OreJson, - RecipeJson, validate_block, validate_item, validate_manifest, @@ -13,32 +10,22 @@ import { validate_ore, validate_recipe, } from "$/common/mod_data.ts"; -import { BUILTIN_MODELS, type ModelJson } from "$/common/block_models.ts"; +import { check_references, load_order, type LoadedMod, type ModReport } from "$/common/mod_check.ts"; +import { png_size } from "$/common/bmod.ts"; -export interface ModReport { - id: string; - errors: string[]; - warnings: string[]; -} - -export interface LoadedMod { - id: string; - dir: string; - report: ModReport; - manifest?: Record; - blocks: { file: string; json: BlockJson }[]; - models: { file: string; json: ModelJson }[]; - items: { file: string; json: ItemJson }[]; - recipes: { file: string; json: RecipeJson }[]; - ores: { file: string; json: OreJson }[]; - textures: string[]; - // absolute paths by texture id - texture_files: Map; -} +export { load_order, type LoadedMod, type ModReport }; // the engine's own textures, named engine: export const ENGINE_TEXTURE_DIR = "assets/sprites/textures"; +// engine:missing is drawn by code, the rest are files +export function engine_texture_ids(): string[] { + return [ + "engine:missing", + ...walk(ENGINE_TEXTURE_DIR, ".png").map((file) => `engine:${file.replace(/\.png$/, "")}`), + ]; +} + // loads and checks every mod without typechecking, which is slow. the build uses this export function load_and_check(mods_dir = "mods"): LoadedMod[] { const mods: LoadedMod[] = []; @@ -47,37 +34,10 @@ export function load_and_check(mods_dir = "mods"): LoadedMod[] { mods.push(load_mod(mods_dir, entry.name)); } } - check_references(mods); + check_references(mods, engine_texture_ids()); return mods; } -// dependencies before the mods that need them, otherwise alphabetical -export function load_order(mods: LoadedMod[]): LoadedMod[] { - const by_id = new Map(mods.map((mod) => [mod.id, mod])); - const ordered: LoadedMod[] = []; - const state = new Map(); - - const visit = (mod: LoadedMod, path: string[]) => { - if (state.get(mod.id) === "done") return; - if (state.get(mod.id) === "visiting") { - throw new Error(`mods depend on each other in a circle: ${[...path, mod.id].join(" -> ")}`); - } - state.set(mod.id, "visiting"); - const dependencies = ((mod.manifest?.dependencies ?? []) as { id: string }[]).map((d) => d.id).sort(); - for (const dependency of dependencies) { - const other = by_id.get(dependency); - if (other) visit(other, [...path, mod.id]); - } - state.set(mod.id, "done"); - ordered.push(mod); - }; - - for (const mod of [...mods].sort((a, b) => a.id.localeCompare(b.id))) { - visit(mod, []); - } - return ordered; -} - export async function check_mods(mods_dir = "mods", options = { typecheck: true }): Promise { const mods = load_and_check(mods_dir); try { @@ -177,7 +137,7 @@ function load_mod(mods_dir: string, id: string): LoadedMod { const texture_id = `${id}:${file.replace(/\.png$/, "").replaceAll("/", "_")}`; mod.textures.push(texture_id); mod.texture_files.set(texture_id, `${dir}/textures/${file}`); - const size = png_size(`${dir}/textures/${file}`); + const size = png_size(Deno.readFileSync(`${dir}/textures/${file}`)); if (!size) { error(`textures/${file}: isn't a png`); } else if (size.width !== 16 || size.height !== 16) { @@ -188,121 +148,16 @@ function load_mod(mods_dir: string, id: string): LoadedMod { return mod; } -function check_references(mods: LoadedMod[]) { - const block_owners = new Map(); - const item_owners = new Map(); - const model_owners = new Map(); - const add = (map: Map, id: string, mod: string) => map.set(id, [...(map.get(id) ?? []), mod]); - - const textures = new Set(); - for (const file of walk(ENGINE_TEXTURE_DIR, ".png")) { - textures.add(`engine:${file.replace(/\.png$/, "")}`); - } - - for (const mod of mods) { - for (const { json } of mod.blocks) { - add(block_owners, json.id, mod.id); - if (json.item !== false) add(item_owners, json.id, mod.id); - } - for (const { json } of mod.items) add(item_owners, json.id, mod.id); - for (const { json } of mod.models) add(model_owners, json.id, mod.id); - for (const texture of mod.textures) textures.add(texture); - } - - const mod_ids = new Set(mods.map((mod) => mod.id)); - - for (const mod of mods) { - const { errors, warnings } = mod.report; - const dependencies = new Set( - ((mod.manifest?.dependencies ?? []) as { id: string }[]).map((dep) => dep.id), - ); - for (const dep of dependencies) { - if (!mod_ids.has(dep)) errors.push(`manifest.json: depends on "${dep}", which isn't installed`); - } - - const uses = (file: string, id: string, what: "block" | "item" | "texture" | "model") => { - const namespace = id.split(":")[0]; - if (namespace !== mod.id && namespace !== "engine" && !dependencies.has(namespace)) { - warnings.push(`${file}: uses ${id} but doesn't list "${namespace}" in dependencies`); - } - if (what === "model") { - if (!BUILTIN_MODELS[id] && !model_owners.has(id)) errors.push(`${file}: model ${id} doesn't exist`); - } else if (what === "texture") { - if (!textures.has(id)) warnings.push(`${file}: texture ${id} doesn't exist, it will show as missing`); - } else if (!(what === "block" ? block_owners : item_owners).has(id)) { - errors.push(`${file}: ${what} ${id} doesn't exist`); - } - }; - - for (const { file, json } of mod.blocks) { - if ((block_owners.get(json.id)?.length ?? 0) > 1) { - errors.push(`${file}: block ${json.id} is also defined by ${block_owners.get(json.id)!.join(", ")}`); - } - for (const variant of [json, ...Object.values(json.variants ?? {})]) { - if (variant.model) uses(file, variant.model, "model"); - const textures = variant.textures ?? {}; - for (const texture of typeof textures === "string" ? [textures] : Object.values(textures)) { - uses(file, texture, "texture"); - } - } - if (json.drops) uses(file, json.drops, "item"); - } - for (const { file, json } of mod.models) { - if ((model_owners.get(json.id)?.length ?? 0) > 1 || BUILTIN_MODELS[json.id]) { - errors.push(`${file}: model ${json.id} is defined more than once`); - } - for (const texture of Object.values(json.textures ?? {})) { - if (!texture.startsWith("#")) uses(file, texture, "texture"); - } - for (const element of json.elements) { - for (const face of Object.values(element.faces)) { - if (face && !face.texture.startsWith("#")) uses(file, face.texture, "texture"); - } - } - } - for (const { file, json } of mod.items) { - if ((item_owners.get(json.id)?.length ?? 0) > 1) { - errors.push(`${file}: item ${json.id} is also defined by ${item_owners.get(json.id)!.join(", ")}`); - } - uses(file, json.texture, "texture"); - if (json.places) uses(file, json.places, "block"); - } - for (const { file, json } of mod.recipes) { - switch (json.type) { - case "shaped": - for (const id of Object.values(json.key)) uses(file, id, "item"); - uses(file, json.result.id, "item"); - break; - case "furnace": - uses(file, json.input, "item"); - uses(file, json.output.id, "item"); - break; - case "fuel": - uses(file, json.item, "item"); - break; - case "smithing": - uses(file, json.tool, "item"); - uses(file, json.material.id, "item"); - if (json.addition) uses(file, json.addition, "item"); - uses(file, json.result, "item"); - break; - } - } - for (const { file, json } of mod.ores) { - uses(file, json.id, "block"); - uses(file, json.replaces, "block"); - } - } -} - async function typecheck_scripts(mod: LoadedMod) { const scripts = Object.values((mod.manifest?.scripts ?? {}) as Record) .filter((path) => typeof path === "string" && exists(`${mod.dir}/${path}`)) .map((path) => `${mod.dir}/${path}`); if (scripts.length === 0) return; + // a mod with its own deno.json is its own deno project, see pack_mod.ts + const config = ["deno.json", "deno.jsonc"].map((name) => `${mod.dir}/${name}`).find(exists) ?? "deno.json"; const output = await new Deno.Command(Deno.execPath(), { - args: ["check", "--config", "deno.json", ...scripts], + args: ["check", "--config", config, ...scripts], stdout: "piped", stderr: "piped", env: { NO_COLOR: "1" }, @@ -361,15 +216,6 @@ function exists(path: string) { } } -// reads the size out of the png header instead of decoding the image -function png_size(path: string): { width: number; height: number } | undefined { - const bytes = Deno.readFileSync(path); - const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; - if (bytes.length < 24 || !signature.every((b, i) => bytes[i] === b)) return undefined; - const view = new DataView(bytes.buffer, bytes.byteOffset); - return { width: view.getUint32(16), height: view.getUint32(20) }; -} - function is_object(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/tools/pack_mod.ts b/tools/pack_mod.ts new file mode 100644 index 0000000..6bf139a --- /dev/null +++ b/tools/pack_mod.ts @@ -0,0 +1,101 @@ +// deno task pack-mod mods/copper_tools [copper_tools.bmod] +// builds a mod folder into a .bmod: its data merged into data.json, each script bundled into one module, and its +// textures and credits. deno task build does this for every mod in mods/, see "Mod files" in MODS.md +import { basename, dirname, resolve } from "@std/path"; +import { type Bmod, BMOD_EXTENSION, SCRIPT_SIDES, type ScriptSide, write_bmod } from "$/common/bmod.ts"; +import type { ManifestJson } from "$/common/mod_data.ts"; +import { load_and_check, type LoadedMod } from "./check_mods.ts"; + +// what each script is bundled for: the server runs in deno, clients in browsers, worldgen in both +const PLATFORMS: Record = { server: "deno", client: "browser", worldgen: "browser" }; + +// a mod that passed check-mods, as .bmod bytes +export async function pack_mod(mod: LoadedMod): Promise> { + const manifest = mod.manifest as unknown as ManifestJson; + const scripts: Bmod["scripts"] = {}; + for (const side of SCRIPT_SIDES) { + const entry = manifest.scripts?.[side]; + if (entry) scripts[side] = await bundle_script(mod.dir, `${mod.dir}/${entry}`, PLATFORMS[side]); + } + + const textures = new Map(); + for (const [id, path] of [...mod.texture_files].sort(([a], [b]) => a.localeCompare(b))) { + textures.set(id, Deno.readFileSync(path)); + } + + return write_bmod({ + manifest, + data: { + blocks: mod.blocks.map((b) => b.json), + models: mod.models.map((m) => m.json), + items: mod.items.map((i) => i.json), + recipes: mod.recipes.map((r) => r.json), + ores: mod.ores.map((o) => o.json), + }, + scripts, + textures, + credits: manifest.credits ? Deno.readTextFileSync(`${mod.dir}/${manifest.credits}`) : undefined, + }); +} + +// a mod is a deno project: with its own deno.json its scripts are bundled with it (its imports, npm packages and +// so on), otherwise with the deno.json of the folder the build runs in +async function bundle_script(mod_dir: string, entry: string, platform: "deno" | "browser"): Promise { + const config = ["deno.json", "deno.jsonc"].map((name) => `${mod_dir}/${name}`).find(exists); + if (!config) { + const result = await Deno.bundle({ entrypoints: [entry], platform, write: false, minify: false }); + if (!result.success || !result.outputFiles?.length) { + throw new Error(`Couldn't bundle ${entry}:\n${result.errors.map((e) => e.text).join("\n")}`); + } + return result.outputFiles[0].text(); + } + + const output = await Deno.makeTempFile({ suffix: ".js" }); + try { + const result = await new Deno.Command(Deno.execPath(), { + args: ["bundle", "--config", config, "--platform", platform, "--output", output, entry], + stdout: "piped", + stderr: "piped", + env: { NO_COLOR: "1" }, + }).output(); + if (!result.success) { + throw new Error(`Couldn't bundle ${entry}:\n${new TextDecoder().decode(result.stderr).trim()}`); + } + return Deno.readTextFileSync(output); + } finally { + Deno.removeSync(output); + } +} + +function exists(path: string) { + try { + Deno.statSync(path); + return true; + } catch { + return false; + } +} + +if (import.meta.main) { + const [dir, out] = Deno.args; + if (!dir) { + console.error("usage: deno task pack-mod [output.bmod]"); + Deno.exit(1); + } + // its siblings are checked too, so references to mods it depends on can be checked + const folder = resolve(dir); + const mod = load_and_check(dirname(folder)).find((m) => resolve(m.dir) === folder); + if (!mod) { + console.error(`${dir} isn't a mod folder`); + Deno.exit(1); + } + if (mod.report.errors.length > 0) { + console.error(`${mod.id} has errors (deno task check-mods for details):`); + for (const error of mod.report.errors) console.error(` ${error}`); + Deno.exit(1); + } + const file = out ?? `${basename(folder)}${BMOD_EXTENSION}`; + const bytes = await pack_mod(mod); + Deno.writeFileSync(file, bytes); + console.log(`Packed ${mod.id} into ${file} (${(bytes.length / 1024).toFixed(1)} KB)`); +}