From e69b554d02e4102e2786c25e51b54b4f650bebb0 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 13 Sep 2025 21:58:43 -0500 Subject: [PATCH] revert --- src/api.js | 379 ++++++++++++++++++++++------------------------------- 1 file changed, 155 insertions(+), 224 deletions(-) diff --git a/src/api.js b/src/api.js index 7a61925..6581e33 100644 --- a/src/api.js +++ b/src/api.js @@ -7,9 +7,8 @@ import { import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; const BUCKET = "repertory"; -const TAR_EXT = ".tar.gz"; -const oldKeys = new Set(); +const oldItems = []; const s3 = new S3Client({ region: "any", @@ -21,101 +20,27 @@ const s3 = new S3Client({ }, }); -/* ---------- helpers ---------- */ - -const toListed = (folderKey) => (o) => { - const d = o.LastModified ?? new Date(0); - const date = d - .toLocaleString("en-US", { - year: "numeric", - month: "2-digit", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - hour12: false, - }) - .replace(/,/g, ""); - return { - date, - sort: d.getTime(), - name: (o.Key ?? "").replace(folderKey, ""), - key: o.Key ?? "", - }; -}; - -const byNewest = (a, b) => b.sort - a.sort; - -const sidecarsFor = (key) => [key + ".sha256", key + ".sig"]; - -// base/packaging helpers -const stripSidecar = (name) => { - if (name.endsWith(".sha256")) return name.slice(0, -".sha256".length); - if (name.endsWith(".sig")) return name.slice(0, -".sig".length); - return name; -}; -const stripPackaging = (name) => { - if (name.endsWith(TAR_EXT)) return name.slice(0, -TAR_EXT.length); - if (name.endsWith("_setup.exe")) return name.slice(0, -"_setup.exe".length); - if (name.endsWith(".dmg")) return name.slice(0, -".dmg".length); - return name; -}; -const baseStem = (name) => stripPackaging(stripSidecar(name)); - -const isTar = (name) => name.endsWith(TAR_EXT); -const isDmg = (name) => name.endsWith(".dmg"); -const isSetup = (name) => name.endsWith("_setup.exe"); - -// normalize + parse FROM base stem so arch is clean -const norm_platform = (p) => { - const v = (p || "").toLowerCase(); - if (v === "macos") return "darwin"; - return v; -}; -const norm_arch = (a) => { - const v = (a || "").toLowerCase(); - if (v === "arm64" || v === "aarch64") return "aarch64"; - if (v === "x86_64" || v === "x86-64") return "x86-64"; - return v; -}; - -// parts: 0=product, 1=version, 2=build, 3=platform, 4=arch -const parseName = (fname) => { - const stem = baseStem(fname); - const parts = stem.split("_"); - const product = parts[0] ?? ""; - const version = parts[1] ?? ""; - const build = parts[2] ?? ""; - const platform = norm_platform(parts[3] ?? ""); - const arch = norm_arch(parts[4] ?? ""); - const groupId = `${product}_${platform}_${arch}`; // retention grouping - const releaseId = `${product}_${build}`; // build/release grouping - return { product, version, build, platform, arch, groupId, releaseId }; -}; - -const platformRank = (p) => - p === "darwin" ? 0 : p === "windows" ? 1 : p === "linux" ? 2 : 3; -const archRank = (a) => (a === "aarch64" ? 0 : a === "x86-64" ? 1 : 2); - -/* ---------- public API ---------- */ - const cleanOldItems = async () => { - const keys = Array.from(oldKeys); - console.log(`cleaning|count|${keys.length}`); - if (keys.length === 0) return; - - const deletes = keys.map((Key) => - s3 - .send(new DeleteObjectCommand({ Bucket: BUCKET, Key })) - .then(() => console.log(`cleaning|key|${Key}`)) - .catch((err) => console.error(`cleaning|error|${Key}`, err)), - ); - await Promise.allSettled(deletes); - oldKeys.clear(); + console.log(`cleaning|count|${oldItems.length}`); + while (oldItems.length > 0) { + try { + const key = oldItems.pop(); + console.log(`cleaning|key|${key}`); + await s3.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: key, + }), + ); + } catch (err) { + console.error(err); + } + } }; const createDownloadLink = async (key) => { - const filename = key.split("/").pop() ?? "download.bin"; + let filename = key.split("/"); + filename = filename[filename.length - 1]; return await getSignedUrl( s3, new GetObjectCommand({ @@ -129,161 +54,167 @@ const createDownloadLink = async (key) => { const getBucketFiles = async (folderName) => { try { - const folder = (folderName ?? "").toLowerCase(); - const folderKey = encodeURIComponent(folder) + "/"; - + folderName = folderName.toLowerCase(); + const folderKey = encodeURIComponent(folderName) + "/"; const data = await s3.send( new ListObjectsCommand({ Bucket: BUCKET, Prefix: folderKey, }), ); - - const contents = (data.Contents ?? []).filter((it) => it.Key !== folderKey); - const listed = contents.map(toListed(folderKey)).sort(byNewest); - - // quick lookups - const byKey = Object.fromEntries(listed.map((i) => [i.key, i])); - - // group by base stem (each base is one platform+arch variant of a build) - const byBase = new Map(); // base -> { rep, tar, setup, dmg, platform, arch, releaseId } - for (const it of listed) { - const base = baseStem(it.name); - let g = byBase.get(base); - if (!g) { - const meta = parseName(it.name); - g = { - rep: it, - tar: null, - setup: null, - dmg: null, - platform: meta.platform, - arch: meta.arch, - releaseId: meta.releaseId, + const ret = data.Contents.filter((item) => item.Key !== folderKey) + .map((item) => { + return { + date: item.LastModified.toLocaleDateString("en-US", { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }).replace(/,/g, ""), + sort: item.LastModified.getTime(), + name: item.Key.replace(folderKey, ""), + key: item.Key, }; - byBase.set(base, g); - } - if (isTar(it.name)) g.tar = it; - else if (isSetup(it.name)) g.setup = it; - else if (isDmg(it.name)) g.dmg = it; - } + }) + .sort((a, b) => { + return a.sort > b.sort ? -1 : a.sort < b.sort ? 1 : 0; + }); + const itemCount = {}; + const ext = ".tar.gz"; - const baseGroups = Array.from(byBase.values()).filter( - (g) => g.tar || g.setup || g.dmg, - ); - - // ---------- Retention: newest 3 per (product+platform+arch) in nightly ---------- - const keepBaseSet = new Set(); - if (folder === "nightly") { - const buckets = new Map(); // groupId -> [bases...] - for (const g of baseGroups) { - const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; - const { groupId } = parseName(rep.name); - if (!buckets.has(groupId)) buckets.set(groupId, []); - buckets.get(groupId).push(g); - } - for (const [, arr] of buckets) { - arr.sort((a, b) => { - const sa = (a.tar ?? a.setup ?? a.dmg ?? a.rep)?.sort ?? 0; - const sb = (b.tar ?? b.setup ?? b.dmg ?? b.rep)?.sort ?? 0; - return sb - sa; - }); - arr.forEach((g, idx) => { - const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; - const base = baseStem(rep.name); - if (idx < 3) { - keepBaseSet.add(base); - } else { - if (g.tar) - [g.tar.key, ...sidecarsFor(g.tar.key)].forEach((k) => - oldKeys.add(k), - ); - if (g.setup) - [g.setup.key, ...sidecarsFor(g.setup.key)].forEach((k) => - oldKeys.add(k), - ); - if (g.dmg) - [g.dmg.key, ...sidecarsFor(g.dmg.key)].forEach((k) => - oldKeys.add(k), - ); + const filteredItems = ret + .filter((item) => item.name.endsWith(ext)) + .filter((item) => { + if (folderName === "nightly") { + const parts = item.name.split("_"); + const groupId = `${parts[0]}_${parts[3]}_${parts[4]}`; + itemCount[groupId] = itemCount[groupId] || 0; + if (++itemCount[groupId] <= 3) { + return true; } - }); - } - } else { - for (const g of baseGroups) { - const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; - keepBaseSet.add(baseStem(rep.name)); - } - } + if (!oldItems.find((key) => key === item.key)) { + // mark tar + sidecars old + oldItems.push(item.key); + oldItems.push(item.key + ".sha256"); + oldItems.push(item.key + ".sig"); - // ---------- Build releases: per release (build) we emit three blocks ---------- - const releases = new Map(); // releaseId -> { repSort, items:[baseGroup] } - for (const g of baseGroups) { - const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; - const base = baseStem(rep.name); - if (!keepBaseSet.has(base)) continue; + // windows companion (_setup.exe) + sidecars + if (parts[3] === "windows") { + const setup_key = + item.key.substring(0, item.key.length - ext.length) + + "_setup.exe"; + const setup_item = ret.find((i) => i.key == setup_key); + if (setup_item) { + oldItems.push(setup_key); + oldItems.push(setup_key + ".sha256"); + oldItems.push(setup_key + ".sig"); + } + } - let r = releases.get(g.releaseId); - if (!r) { - r = { repSort: rep.sort, items: [] }; - releases.set(g.releaseId, r); - } - r.repSort = Math.max(r.repSort, rep.sort); - r.items.push({ ...g, repSort: rep.sort }); - } - - // newest releases first - const releaseList = Array.from(releases.entries()).sort( - (a, b) => b[1].repSort - a[1].repSort, - ); - - const out = []; - for (const [, bucket] of releaseList) { - // Sort items consistently for all three blocks - const sorted = bucket.items.slice().sort((a, b) => { - const pr = platformRank(a.platform) - platformRank(b.platform); - if (pr !== 0) return pr; - const ar = archRank(a.arch) - archRank(b.arch); - if (ar !== 0) return ar; - return b.repSort - a.repSort; // tie-breaker newest first + // darwin companion (.dmg) + sidecars <-- NEW + if (parts[3] === "darwin") { + const dmg_key = + item.key.substring(0, item.key.length - ext.length) + ".dmg"; + const dmg_item = ret.find((i) => i.key == dmg_key); + if (dmg_item) { + oldItems.push(dmg_key); + oldItems.push(dmg_key + ".sha256"); + oldItems.push(dmg_key + ".sig"); + } + } + } + return false; + } + return true; }); - // Block 1: all tar.gz (+ sidecars) - for (const g of sorted) { - if (!g.tar) continue; - out.push(g.tar); - for (const sk of sidecarsFor(g.tar.key)) { - const it = byKey[sk]; - if (it) out.push(it); + const totalCount = filteredItems.length * 3; + const setup_items = []; + const dmg_items = []; // <-- NEW + let setup_offset = 0; + let dmg_offset = 0; + + for (let i = 0; i < totalCount && i < filteredItems.length; i += 3) { + const parts = filteredItems[i].name.split("_"); + + // add tar sidecars + let item = ret.filter( + (it) => it.name === filteredItems[i].name + ".sha256", + ); + filteredItems.splice(i + 1, 0, ...item); + item = ret.filter((it) => it.name === filteredItems[i].name + ".sig"); + filteredItems.splice(i + 2, 0, ...item); + + // windows companion block (setup.exe + sidecars) + if (parts[3] === "windows") { + const setup_key = + filteredItems[i].key.substring( + 0, + filteredItems[i].key.length - ext.length, + ) + "_setup.exe"; + + const setup_item = ret.find((it) => it.key == setup_key); + if (setup_item) { + const setup_item2 = ret.find((it) => it.key == setup_key + ".sha256"); + const setup_item3 = ret.find((it) => it.key == setup_key + ".sig"); + setup_items.push([ + { idx: i + 3 + setup_offset }, + setup_item, + setup_item2, + setup_item3, + ]); + setup_offset += 3; } } - // Block 2: all _setup.exe (+ sidecars) - for (const g of sorted) { - if (!g.setup) continue; - out.push(g.setup); - for (const sk of sidecarsFor(g.setup.key)) { - const it = byKey[sk]; - if (it) out.push(it); - } - } + // darwin companion block (.dmg + sidecars) <-- NEW (mirrors setup flow) + if (parts[3] === "darwin") { + const dmg_key = + filteredItems[i].key.substring( + 0, + filteredItems[i].key.length - ext.length, + ) + ".dmg"; - // Block 3: all .dmg (+ sidecars) - for (const g of sorted) { - if (!g.dmg) continue; - out.push(g.dmg); - for (const sk of sidecarsFor(g.dmg.key)) { - const it = byKey[sk]; - if (it) out.push(it); + const dmg_item = ret.find((it) => it.key == dmg_key); + if (dmg_item) { + const dmg_item2 = ret.find((it) => it.key == dmg_key + ".sha256"); + const dmg_item3 = ret.find((it) => it.key == dmg_key + ".sig"); + dmg_items.push([ + { idx: i + 3 + dmg_offset }, + dmg_item, + dmg_item2, + dmg_item3, + ]); + dmg_offset += 3; } } } - return out; + // Insert darwin DMG blocks first, then Windows setup blocks (order doesn't matter across platforms) + if (dmg_items.length > 0) { + dmg_items.forEach((items) => { + filteredItems.splice(items[0].idx, 0, items[1]); + filteredItems.splice(items[0].idx + 1, 0, items[2]); + filteredItems.splice(items[0].idx + 2, 0, items[3]); + }); + } + + if (setup_items.length > 0) { + setup_items.forEach((items) => { + filteredItems.splice(items[0].idx, 0, items[1]); + filteredItems.splice(items[0].idx + 1, 0, items[2]); + filteredItems.splice(items[0].idx + 2, 0, items[3]); + }); + } + + return filteredItems; } catch (err) { console.error(err); - return []; } + + return []; }; export { cleanOldItems, createDownloadLink, getBucketFiles };