From 7cc5e98cfbd735945d969fc9336020e66a176eb4 Mon Sep 17 00:00:00 2001 From: "Scott E. Graves" Date: Sat, 13 Sep 2025 21:33:22 -0500 Subject: [PATCH] added dmg --- src/api.js | 259 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 147 insertions(+), 112 deletions(-) diff --git a/src/api.js b/src/api.js index 1c726c8..1ddeac9 100644 --- a/src/api.js +++ b/src/api.js @@ -7,8 +7,9 @@ import { import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; const BUCKET = "repertory"; +const TAR_EXT = ".tar.gz"; -const oldItems = []; +const oldKeys = new Set(); const s3 = new S3Client({ region: "any", @@ -20,27 +21,76 @@ const s3 = new S3Client({ }, }); +/** Helpers **/ + +const toListed = (folderKey) => (o) => { + const d = o.LastModified ?? new Date(0); + const date = d + .toLocaleString("en-US", { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + }) + .replace(/,/g, ""); + return { + date, + sort: d.getTime(), + name: (o.Key ?? "").replace(folderKey, ""), + key: o.Key ?? "", + }; +}; + +const byNewest = (a, b) => b.sort - a.sort; + +const sidecarsFor = (key) => [key + ".sha256", key + ".sig"]; + +const parseName = (fname) => { + // Expected pattern: parts[3] = platform (windows|darwin|linux), parts[4] = arch + const parts = fname.split("_"); + const platform = parts[3] ?? ""; + const arch = parts[4] ?? ""; + const groupId = `${parts[0] ?? ""}_${platform}_${arch}`; + return { parts, platform, arch, groupId }; +}; + +const windowsCompanionFromTar = (tarKey) => { + // .../name.tar.gz -> .../name_setup.exe + const base = tarKey.slice(0, -TAR_EXT.length); + return `${base}_setup.exe`; +}; + +const darwinCompanionFromTar = (tarKey) => { + // .../name.tar.gz -> .../name.dmg + const base = tarKey.slice(0, -TAR_EXT.length); + return `${base}.dmg`; +}; + +/** Public API **/ + const cleanOldItems = async () => { - console.log(`cleaning|count|${oldItems.length}`); - while (oldItems.length > 0) { - try { - const key = oldItems.pop(); - console.log(`cleaning|key|${key}`); - await s3.send( - new DeleteObjectCommand({ - Bucket: BUCKET, - Key: key, - }), - ); - } catch (err) { - console.error(err); - } + const keys = Array.from(oldKeys); + console.log(`cleaning|count|${keys.length}`); + if (keys.length === 0) { + return; } + + const deletes = keys.map((Key) => + s3 + .send(new DeleteObjectCommand({ Bucket: BUCKET, Key })) + .then(() => console.log(`cleaning|key|${Key}`)) + .catch((err) => console.error(`cleaning|error|${Key}`, err)), + ); + + await Promise.allSettled(deletes); + oldKeys.clear(); }; const createDownloadLink = async (key) => { - let filename = key.split("/"); - filename = filename[filename.length - 1]; + const filename = key.split("/").pop() ?? "download.bin"; return await getSignedUrl( s3, new GetObjectCommand({ @@ -54,121 +104,106 @@ const createDownloadLink = async (key) => { const getBucketFiles = async (folderName) => { try { - folderName = folderName.toLowerCase(); - const folderKey = encodeURIComponent(folderName) + "/"; + const folder = (folderName ?? "").toLowerCase(); + const folderKey = encodeURIComponent(folder) + "/"; + const data = await s3.send( new ListObjectsCommand({ Bucket: BUCKET, Prefix: folderKey, }), ); - const ret = data.Contents.filter((item) => item.Key !== folderKey) - .map((item) => { - return { - date: item.LastModified.toLocaleDateString("en-US", { - year: "numeric", - month: "2-digit", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - }).replace(/,/g, ""), - sort: item.LastModified.getTime(), - name: item.Key.replace(folderKey, ""), - key: item.Key, - }; - }) - .sort((a, b) => { - return a.sort > b.sort ? -1 : a.sort < b.sort ? 1 : 0; - }); - const itemCount = {}; - const ext = ".tar.gz"; - const filteredItems = ret - .filter((item) => item.name.endsWith(ext)) - .filter((item) => { - if (folderName === "nightly") { - const parts = item.name.split("_"); - const groupId = `${parts[0]}_${parts[3]}_${parts[4]}`; - itemCount[groupId] = itemCount[groupId] || 0; - if (++itemCount[groupId] <= 3) { - return true; + const contents = (data.Contents ?? []).filter((it) => it.Key !== folderKey); + + // Normalize listing + const listed = contents.map(toListed(folderKey)).sort(byNewest); + + // Consider only .tar.gz for retention policy + const tarItems = listed.filter((i) => i.name.endsWith(TAR_EXT)); + + // Nightly policy: keep top 3 per (name_platform_arch); mark others + companions for deletion + const keepSet = new Set(); + if (folder === "nightly") { + const groupCounts = {}; + for (const t of tarItems) { + const { groupId, platform } = parseName(t.name); + groupCounts[groupId] = (groupCounts[groupId] ?? 0) + 1; + const count = groupCounts[groupId]; + + if (count <= 3) { + keepSet.add(t.key); + } else { + [t.key, ...sidecarsFor(t.key)].forEach((k) => oldKeys.add(k)); + + if (platform === "windows") { + const setupKey = windowsCompanionFromTar(t.key); + [setupKey, ...sidecarsFor(setupKey)].forEach((k) => oldKeys.add(k)); + } else if (platform === "darwin") { + const dmgKey = darwinCompanionFromTar(t.key); + [dmgKey, ...sidecarsFor(dmgKey)].forEach((k) => oldKeys.add(k)); } - if (!oldItems.find((key) => key === item.key)) { - oldItems.push(item.key); - oldItems.push(item.key + ".sha256"); - oldItems.push(item.key + ".sig"); + } + } + } else { + tarItems.forEach((t) => keepSet.add(t.key)); + } - if (parts[3] === "windows") { - const setup_key = - item.key.substring(0, item.key.length - ext.length) + - "_setup.exe"; + // Build final output list (tar + sidecars + optional platform companion + its sidecars) + const byKey = Object.fromEntries(listed.map((i) => [i.key, i])); + const out = []; - const setup_item = ret.find((item) => item.key == setup_key); - if (setup_item) { - oldItems.push(setup_key); - oldItems.push(setup_key + ".sha256"); - oldItems.push(setup_key + ".sig"); - } + for (const t of tarItems) { + if (!keepSet.has(t.key)) { + continue; + } + + // tar.gz + out.push(t); + + // tar sidecars + for (const sk of sidecarsFor(t.key)) { + const it = byKey[sk]; + if (it) { + out.push(it); + } + } + + // platform companions + const { platform } = parseName(t.name); + + if (platform === "windows") { + const setupKey = windowsCompanionFromTar(t.key); + const setupItem = byKey[setupKey]; + if (setupItem) { + out.push(setupItem); + for (const sk of sidecarsFor(setupKey)) { + const it = byKey[sk]; + if (it) { + out.push(it); } } - return false; } - return true; - }); - - const totalCount = filteredItems.length * 3; - const setup_items = []; - let setup_offset = 0; - for (let i = 0; i < totalCount && i < filteredItems.length; i += 3) { - const parts = filteredItems[i].name.split("_"); - let item = ret.filter( - (item) => item.name === filteredItems[i].name + ".sha256", - ); - filteredItems.splice(i + 1, 0, ...item); - item = ret.filter((item) => item.name === filteredItems[i].name + ".sig"); - filteredItems.splice(i + 2, 0, ...item); - - if (parts[3] === "windows") { - const setup_key = - filteredItems[i].key.substring( - 0, - filteredItems[i].key.length - ext.length, - ) + "_setup.exe"; - - const setup_item = ret.find((item) => item.key == setup_key); - if (setup_item) { - const setup_item2 = ret.find( - (item) => item.key == setup_key + ".sha256", - ); - const setup_item3 = ret.find( - (item) => item.key == setup_key + ".sig", - ); - setup_items.push([ - { idx: i + 3 + setup_offset }, - setup_item, - setup_item2, - setup_item3, - ]); - setup_offset += 3; + } else if (platform === "darwin") { + const dmgKey = darwinCompanionFromTar(t.key); + const dmgItem = byKey[dmgKey]; + if (dmgItem) { + out.push(dmgItem); + for (const sk of sidecarsFor(dmgKey)) { + const it = byKey[sk]; + if (it) { + out.push(it); + } + } } } } - if (setup_items.length > 0) { - setup_items.forEach((items) => { - filteredItems.splice(items[0].idx, 0, items[1]); - filteredItems.splice(items[0].idx + 1, 0, items[2]); - filteredItems.splice(items[0].idx + 2, 0, items[3]); - }); - } - - return filteredItems; + return out; } catch (err) { console.error(err); + return []; } - - return []; }; export { cleanOldItems, createDownloadLink, getBucketFiles };