diff options
| -rw-r--r-- | public/admin.css | 60 | ||||
| -rw-r--r-- | public/admin.html | 17 | ||||
| -rw-r--r-- | public/admin.js | 53 | ||||
| -rw-r--r-- | server.mjs | 184 |
4 files changed, 308 insertions, 6 deletions
diff --git a/public/admin.css b/public/admin.css index 8f95d63..8756359 100644 --- a/public/admin.css +++ b/public/admin.css @@ -159,6 +159,66 @@ button.danger:hover { border-color: var(--danger); background: rgba(252, 165, 16 margin: 0 0 8px; } +.local { + background: var(--panel); + border: 1px solid rgba(255, 255, 255, .08); + border-radius: 10px; + padding: 14px; + display: grid; + gap: 10px; +} +.local h2 { + font-size: 14px; + font-weight: 500; + letter-spacing: .08em; + text-transform: uppercase; + color: var(--muted); + margin: 0; +} +.local-head { display: flex; align-items: center; justify-content: space-between; gap: 10px; } +.local-head h2 { margin: 0; } +.local-path { + margin: 0; + font-size: 12px; + color: var(--ring); + font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace; + overflow-wrap: anywhere; +} +.local-hint { margin: 0; font-size: 13px; color: var(--muted); line-height: 1.5; } +.local-hint code { + display: inline-block; + margin-top: 4px; + font-size: 12px; + color: var(--ink); + background: var(--panel-2); + border: 1px solid rgba(255, 255, 255, .1); + border-radius: 6px; + padding: 3px 8px; + overflow-wrap: anywhere; +} +.st-file { + display: flex; + justify-content: space-between; + gap: 10px; + padding: 6px 8px; + background: var(--panel-2); + border: 1px solid rgba(255, 255, 255, .06); + border-radius: 8px; + font-size: 13px; +} +.st-file .n { color: var(--ink); overflow-wrap: anywhere; } +.st-file .s { color: var(--muted); white-space: nowrap; } +.st-log-item { + padding: 6px 8px; + font-size: 12px; + border-radius: 8px; + background: var(--panel-2); + border: 1px solid rgba(255, 255, 255, .06); +} +.st-log-item.ok { color: var(--ok); } +.st-log-item.bad { color: var(--danger); } +#st-log { display: grid; gap: 6px; } + .results { display: grid; grid-template-columns: repeat(auto-fill, minmax(110px, 1fr)); diff --git a/public/admin.html b/public/admin.html index 9fe1b5a..c296bea 100644 --- a/public/admin.html +++ b/public/admin.html @@ -5,7 +5,7 @@ <meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="color-scheme" content="dark"> <title>pathways · admin</title> - <link rel="stylesheet" href="/admin.css?v=4"> + <link rel="stylesheet" href="/admin.css?v=5"> </head> <body> <main id="app"> @@ -73,6 +73,19 @@ <p class="im-status" id="im-status"></p> </div> <p class="err" id="im-err"></p> + <div class="local"> + <div class="local-head"> + <h2>local drop folder</h2> + <button id="st-scan">import now</button> + </div> + <p class="local-path" id="st-path"></p> + <p class="local-hint">copy photos into this folder on the server — they import automatically (polled every ~15s).<br> + <code>scp photo.jpg dax@<host>:/srv/pathways/staging/watch/</code></p> + <div id="st-files"></div> + <p class="facets" id="st-none" hidden>no files staged yet</p> + <h2>recent imports</h2> + <div id="st-log"></div> + </div> <div class="recent" id="im-recent" hidden> <h2>recently added</h2> <div class="results" id="im-new"></div> @@ -87,6 +100,6 @@ </section> </main> <div id="toast"></div> - <script src="/admin.js?v=4"></script> + <script src="/admin.js?v=5"></script> </body> </html> diff --git a/public/admin.js b/public/admin.js index 90dd6b0..5291620 100644 --- a/public/admin.js +++ b/public/admin.js @@ -99,6 +99,13 @@ function fmtUptime(s) { return h ? `${h} h ${m} m` : `${m} m` } +function fmtSize(n) { + if (n >= 1e9) return (n / 1e9).toFixed(1) + " GB" + if (n >= 1e6) return (n / 1e6).toFixed(1) + " MB" + if (n >= 1e3) return Math.round(n / 1e3) + " KB" + return n + " B" +} + let debounceTimer = 0 function debounce(fn, ms) { clearTimeout(debounceTimer) @@ -294,6 +301,7 @@ async function enterDash() { try { await loadStatus() await loadRecent() + await loadStaging().catch(() => {}) } catch (err) { if (views.dash.hidden) return; } } @@ -316,6 +324,34 @@ async function loadRecent() { $("#im-recent").hidden = data.photos.length === 0 } +async function loadStaging() { + const d = await api("/api/admin/staging") + $("#st-path").textContent = d.watch + const filesEl = $("#st-files") + filesEl.replaceChildren() + for (const f of d.files) { + const row = document.createElement("div") + row.className = "st-file" + const n = document.createElement("span") + n.className = "n" + n.textContent = f.name + const s = document.createElement("span") + s.className = "s" + s.textContent = fmtSize(f.size) + " · " + fmtAge(Date.now() - f.mtime) + row.append(n, s) + filesEl.appendChild(row) + } + $("#st-none").hidden = d.files.length > 0 + const logEl = $("#st-log") + logEl.replaceChildren() + for (const r of d.imports) { + const row = document.createElement("div") + row.className = "st-log-item " + (r.ok ? "ok" : "bad") + row.textContent = `${new Date(r.at).toLocaleString()} — ${r.ok ? `imported ${r.count} ${r.count === 1 ? "photo" : "photos"}` : "failed: " + (r.error || "unknown error")}` + logEl.appendChild(row) + } +} + function setImProgress(pct, text) { $("#im-progress").hidden = false $("#im-status").textContent = text @@ -360,6 +396,7 @@ async function importFiles(list) { toast(`imported ${data.imported} ${data.imported === 1 ? "photo" : "photos"}`) await loadStatus() await loadRecent() + await loadStaging().catch(() => {}) } catch (err) { $("#im-err").textContent = err.message } finally { @@ -378,10 +415,24 @@ async function init() { $("#tab-status").hidden = tab !== "status" $("#tab-import").hidden = tab !== "import" $("#tab-graph").hidden = tab !== "graph" - if (tab === "import") loadRecent().catch((e2) => toast(e2.message)) + if (tab === "import") { + loadRecent().catch((e2) => toast(e2.message)) + loadStaging().catch((e2) => toast(e2.message)) + } }) }) + $("#st-scan").addEventListener("click", async () => { + const btn = $("#st-scan") + btn.disabled = true + try { + await api("/api/admin/staging", { method: "POST" }) + await loadStaging() + toast("scan complete") + } catch (err) { toast(err.message) } + btn.disabled = false + }) + $("#setup-form").addEventListener("submit", async (e) => { e.preventDefault() const err = $("#setup-err") @@ -1,5 +1,5 @@ import http from "node:http" -import { open, mkdir, rm, rename, readFile, writeFile } from "node:fs/promises" +import { open, mkdir, rm, rename, readFile, writeFile, readdir, stat } from "node:fs/promises" import { createReadStream, createWriteStream, existsSync } from "node:fs" import { extname, join, normalize } from "node:path" import { execFile } from "node:child_process" @@ -37,6 +37,17 @@ const MAX_UPLOAD_BYTES = Number(process.env.MAX_UPLOAD_BYTES || 1024 * 1024 * 10 const MAX_UPLOAD_FILES = Number(process.env.MAX_UPLOAD_FILES || 100) const IMPORT_TIMEOUT_MS = Number(process.env.IMPORT_TIMEOUT_MS || 30 * 60 * 1000) +const WATCH_ROOT = process.env.WATCH_ROOT || join(STAGING_ROOT, "watch") +const FAILED_ROOT = join(STAGING_ROOT, "failed") +const WATCH_POLL_MS = Number(process.env.WATCH_POLL_MS || 15_000) +const WATCH_SETTLE_MS = Number(process.env.WATCH_SETTLE_MS || 20_000) +const BATCH_MAX_AGE_MS = Number(process.env.BATCH_MAX_AGE_MS || 12 * 3600 * 1000) +const FAILED_MAX_AGE_MS = Number(process.env.FAILED_MAX_AGE_MS || 7 * 24 * 3600 * 1000) +const WATCH_EXTENSIONS = new Set([ + ".jpg", ".jpeg", ".png", ".webp", ".gif", ".tif", ".tiff", + ".heic", ".heif", ".dng", ".cr2", ".nef", ".arw", ".orf", ".rw2", ".raf", ".raw", +]) + function directOrigins() { const lan = [] const tailscale = [] @@ -489,9 +500,162 @@ function photoprismCopy(dir) { return execFileAsync("sudo", args, { cwd: PHOTOPRISM_HOME, timeout: IMPORT_TIMEOUT_MS, maxBuffer: 16 * 1024 * 1024 }) } +const importLog = [] + +function logImport(entry) { + importLog.unshift(entry) + if (importLog.length > 12) importLog.pop() +} + +let watchBusy = false + +async function listFiles(dir, base = "") { + const out = [] + let entries + try { entries = await readdir(dir, { withFileTypes: true }) } catch { return out } + for (const e of entries) { + if (e.name.startsWith(".")) continue + const rel = base ? `${base}/${e.name}` : e.name + if (e.isDirectory()) out.push(...await listFiles(join(dir, e.name), rel)) + else if (e.isFile()) out.push(rel) + } + return out +} + +async function watchFiles() { + const files = [] + for (const rel of await listFiles(WATCH_ROOT)) { + const st = await stat(join(WATCH_ROOT, rel)).catch(() => null) + if (st?.isFile()) files.push({ name: rel, size: st.size, mtime: st.mtimeMs }) + } + return files +} + +async function pruneEmptyDirs(dir) { + let entries = [] + try { entries = await readdir(dir, { withFileTypes: true }) } catch { return } + for (const e of entries) { + if (!e.isDirectory()) continue + const sub = join(dir, e.name) + await pruneEmptyDirs(sub) + try { + if ((await readdir(sub)).length === 0) { + await rm(sub, { recursive: true, force: true }) + console.log(`pathways: watch removed empty folder ${sub}`) + } + } catch { /* ignore */ } + } +} + +async function runWatchImport() { + if (watchBusy) return + watchBusy = true + try { + await mkdir(WATCH_ROOT, { recursive: true }) + const now = Date.now() + const staged = [] + const leftovers = [] + for (const rel of await listFiles(WATCH_ROOT)) { + const st = await stat(join(WATCH_ROOT, rel)).catch(() => null) + if (!st || !st.isFile()) continue + const ext = extname(rel).toLowerCase() + if (ext === ".tmp" || ext === ".part" || ext === ".partial") { + leftovers.push(rel) + continue + } + if (now - st.mtimeMs < WATCH_SETTLE_MS) continue + if (!WATCH_EXTENSIONS.has(ext)) continue + staged.push({ rel, src: join(WATCH_ROOT, rel), size: st.size }) + } + for (const rel of leftovers) { + await rm(join(WATCH_ROOT, rel), { force: true }).catch(() => {}) + console.log(`pathways: watch removed leftover part file ${rel}`) + } + await pruneEmptyDirs(WATCH_ROOT) + if (!staged.length) return + + const batch = randomUUID() + const dir = join(STAGING_ROOT, batch) + await mkdir(dir, { recursive: true }) + const moved = [] + for (const f of staged) { + const safe = f.rel.split("/").pop().replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 120) || "upload" + let out = safe + let n = 1 + while (moved.some((m) => m.name === out)) { + const dot = safe.lastIndexOf(".") + out = `${safe.slice(0, dot)}-${n}${safe.slice(dot)}` + n++ + } + await rename(f.src, join(dir, out)).catch(() => {}) + moved.push({ name: out, size: f.size }) + } + if (!moved.length) { + await rm(dir, { recursive: true, force: true }).catch(() => {}) + return + } + + try { + const { stdout, stderr } = await photoprismCopy(dir) + if (stderr) console.error(`pathways: watch import stderr: ${stderr.slice(-2000)}`) + else if (stdout) console.log(`pathways: watch import: ${stdout.slice(-2000)}`) + await refresh() + await rm(dir, { recursive: true, force: true }).catch(() => {}) + logImport({ at: new Date().toISOString(), count: moved.length, ok: true }) + console.log(`pathways: watch imported ${moved.length} file(s)`) + } catch (err) { + const failDir = join(FAILED_ROOT, batch) + await mkdir(FAILED_ROOT, { recursive: true }) + await rename(dir, failDir).catch(async () => { await rm(dir, { recursive: true, force: true }).catch(() => {}) }) + logImport({ at: new Date().toISOString(), count: moved.length, ok: false, error: (err.stderr || err.message || "photoprism import failed").toString().slice(0, 300) }) + console.error(`pathways: watch import failed (batch ${batch}):`, (err.stderr || err.message || "").toString().slice(-2000)) + } + } catch (err) { + console.error("pathways: watch poll error:", err) + } finally { + watchBusy = false + } +} + +async function recoverStaging() { + await mkdir(WATCH_ROOT, { recursive: true }) + await mkdir(FAILED_ROOT, { recursive: true }) + const now = Date.now() + let entries = [] + try { entries = await readdir(STAGING_ROOT, { withFileTypes: true }) } catch { return } + for (const e of entries) { + if (!e.isDirectory() || e.name === "watch" || e.name === "failed") continue + const dir = join(STAGING_ROOT, e.name) + const st = await stat(dir).catch(() => null) + if (!st) continue + if (now - st.mtimeMs < BATCH_MAX_AGE_MS) { + await rename(dir, join(WATCH_ROOT, `recovered-${e.name.slice(0, 8)}`)).catch(() => {}) + console.log(`pathways: recovered interrupted batch ${e.name}`) + } else { + await rm(dir, { recursive: true, force: true }).catch(() => {}) + console.log(`pathways: purged stale batch ${e.name}`) + } + } + let failed = [] + try { failed = await readdir(FAILED_ROOT, { withFileTypes: true }) } catch { return } + for (const e of failed) { + if (!e.isDirectory()) continue + const dir = join(FAILED_ROOT, e.name) + const st = await stat(dir).catch(() => null) + if (st && now - st.mtimeMs > FAILED_MAX_AGE_MS) { + await rm(dir, { recursive: true, force: true }).catch(() => {}) + } + } +} + async function handleImport(req, res) { if (req.method !== "POST") return json(res, 405, { error: "method not allowed" }) + const contentLength = Number(req.headers["content-length"] || 0) + if (contentLength > MAX_UPLOAD_BYTES) { + return json(res, 413, { error: `upload too large (max ${MAX_UPLOAD_BYTES} bytes)` }) + } + const ctype = req.headers["content-type"] || "" const bm = /boundary=(?:"([^"]+)"|([^;]+))/i.exec(ctype) if (!/^multipart\/form-data/i.test(ctype) || !bm) { @@ -703,6 +867,15 @@ const server = http.createServer(async (req, res) => { return json(res, 200, { id, dirs }) } + if (path === "/api/admin/staging") { + if (req.method === "GET" || req.method === "POST") { + if (req.method === "POST") await runWatchImport() + const files = await watchFiles() + return json(res, 200, { watch: WATCH_ROOT, files, imports: importLog }) + } + return json(res, 405, { error: "method not allowed" }) + } + if (path === "/api/import") { const admin = await loadAdmin() if (!admin) return json(res, 401, { error: "no admin password configured; visit /admin first" }) @@ -734,7 +907,12 @@ server.keepAliveTimeout = 65_000 server.listen(PORT, "127.0.0.1", () => { console.log(`pathways listening on 127.0.0.1:${PORT}`) - mkdir(STAGING_ROOT, { recursive: true }).catch((err) => console.error("pathways: staging dir:", err.message)) - refresh() + mkdir(WATCH_ROOT, { recursive: true }) + .then(() => mkdir(FAILED_ROOT, { recursive: true })) + .then(recoverStaging) + .then(runWatchImport) + .then(() => refresh()) + .catch((err) => console.error("pathways: watch init:", err.message)) }) setInterval(refresh, 30 * 60 * 1000).unref() +setInterval(runWatchImport, WATCH_POLL_MS).unref() |