Public/fifthgrid_browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revert

Browse Source
This commit is contained in:
Scott E. Graves
2025-09-13 21:58:43 -05:00
parent 3aaa8ea682
commit e69b554d02
1 changed files with 155 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

371
src/api.js
View File

@@ -7,9 +7,8 @@ import {
import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
const BUCKET = "repertory"; const BUCKET = "repertory";
const TAR_EXT = ".tar.gz";
const oldKeys = new Set(); const oldItems = [];
const s3 = new S3Client({ const s3 = new S3Client({
region: "any", region: "any",
@@ -21,101 +20,27 @@ const s3 = new S3Client({
}, },
}); });
/* ---------- helpers ---------- */
const toListed = (folderKey) => (o) => {
const d = o.LastModified ?? new Date(0);
const date = d
.toLocaleString("en-US", {
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
hour12: false,
})
.replace(/,/g, "");
return {
date,
sort: d.getTime(),
name: (o.Key ?? "").replace(folderKey, ""),
key: o.Key ?? "",
};
};
const byNewest = (a, b) => b.sort - a.sort;
const sidecarsFor = (key) => [key + ".sha256", key + ".sig"];
// base/packaging helpers
const stripSidecar = (name) => {
if (name.endsWith(".sha256")) return name.slice(0, -".sha256".length);
if (name.endsWith(".sig")) return name.slice(0, -".sig".length);
return name;
};
const stripPackaging = (name) => {
if (name.endsWith(TAR_EXT)) return name.slice(0, -TAR_EXT.length);
if (name.endsWith("_setup.exe")) return name.slice(0, -"_setup.exe".length);
if (name.endsWith(".dmg")) return name.slice(0, -".dmg".length);
return name;
};
const baseStem = (name) => stripPackaging(stripSidecar(name));
const isTar = (name) => name.endsWith(TAR_EXT);
const isDmg = (name) => name.endsWith(".dmg");
const isSetup = (name) => name.endsWith("_setup.exe");
// normalize + parse FROM base stem so arch is clean
const norm_platform = (p) => {
const v = (p || "").toLowerCase();
if (v === "macos") return "darwin";
return v;
};
const norm_arch = (a) => {
const v = (a || "").toLowerCase();
if (v === "arm64" || v === "aarch64") return "aarch64";
if (v === "x86_64" || v === "x86-64") return "x86-64";
return v;
};
// parts: 0=product, 1=version, 2=build, 3=platform, 4=arch
const parseName = (fname) => {
const stem = baseStem(fname);
const parts = stem.split("_");
const product = parts[0] ?? "";
const version = parts[1] ?? "";
const build = parts[2] ?? "";
const platform = norm_platform(parts[3] ?? "");
const arch = norm_arch(parts[4] ?? "");
const groupId = `${product}_${platform}_${arch}`; // retention grouping
const releaseId = `${product}_${build}`; // build/release grouping
return { product, version, build, platform, arch, groupId, releaseId };
};
const platformRank = (p) =>
p === "darwin" ? 0 : p === "windows" ? 1 : p === "linux" ? 2 : 3;
const archRank = (a) => (a === "aarch64" ? 0 : a === "x86-64" ? 1 : 2);
/* ---------- public API ---------- */
const cleanOldItems = async () => { const cleanOldItems = async () => {
const keys = Array.from(oldKeys); console.log(`cleaning|count|${oldItems.length}`);
console.log(`cleaning|count|${keys.length}`); while (oldItems.length > 0) {
if (keys.length === 0) return; try {
const key = oldItems.pop();
const deletes = keys.map((Key) => console.log(`cleaning|key|${key}`);
s3 await s3.send(
.send(new DeleteObjectCommand({ Bucket: BUCKET, Key })) new DeleteObjectCommand({
.then(() => console.log(`cleaning|key|${Key}`)) Bucket: BUCKET,
.catch((err) => console.error(`cleaning|error|${Key}`, err)), Key: key,
}),
); );
await Promise.allSettled(deletes); } catch (err) {
oldKeys.clear(); console.error(err);
}
}
}; };
const createDownloadLink = async (key) => { const createDownloadLink = async (key) => {
const filename = key.split("/").pop() ?? "download.bin"; let filename = key.split("/");
filename = filename[filename.length - 1];
return await getSignedUrl( return await getSignedUrl(
s3, s3,
new GetObjectCommand({ new GetObjectCommand({
@@ -129,161 +54,167 @@ const createDownloadLink = async (key) => {
const getBucketFiles = async (folderName) => { const getBucketFiles = async (folderName) => {
try { try {
const folder = (folderName ?? "").toLowerCase(); folderName = folderName.toLowerCase();
const folderKey = encodeURIComponent(folder) + "/"; const folderKey = encodeURIComponent(folderName) + "/";
const data = await s3.send( const data = await s3.send(
new ListObjectsCommand({ new ListObjectsCommand({
Bucket: BUCKET, Bucket: BUCKET,
Prefix: folderKey, Prefix: folderKey,
}), }),
); );
const ret = data.Contents.filter((item) => item.Key !== folderKey)
const contents = (data.Contents ?? []).filter((it) => it.Key !== folderKey); .map((item) => {
const listed = contents.map(toListed(folderKey)).sort(byNewest); return {
date: item.LastModified.toLocaleDateString("en-US", {
// quick lookups year: "numeric",
const byKey = Object.fromEntries(listed.map((i) => [i.key, i])); month: "2-digit",
day: "2-digit",
// group by base stem (each base is one platform+arch variant of a build) hour: "2-digit",
const byBase = new Map(); // base -> { rep, tar, setup, dmg, platform, arch, releaseId } minute: "2-digit",
for (const it of listed) { second: "2-digit",
const base = baseStem(it.name); }).replace(/,/g, ""),
let g = byBase.get(base); sort: item.LastModified.getTime(),
if (!g) { name: item.Key.replace(folderKey, ""),
const meta = parseName(it.name); key: item.Key,
g = {
rep: it,
tar: null,
setup: null,
dmg: null,
platform: meta.platform,
arch: meta.arch,
releaseId: meta.releaseId,
}; };
byBase.set(base, g); })
} .sort((a, b) => {
if (isTar(it.name)) g.tar = it; return a.sort > b.sort ? -1 : a.sort < b.sort ? 1 : 0;
else if (isSetup(it.name)) g.setup = it;
else if (isDmg(it.name)) g.dmg = it;
}
const baseGroups = Array.from(byBase.values()).filter(
(g) => g.tar || g.setup || g.dmg,
);
// ---------- Retention: newest 3 per (product+platform+arch) in nightly ----------
const keepBaseSet = new Set();
if (folder === "nightly") {
const buckets = new Map(); // groupId -> [bases...]
for (const g of baseGroups) {
const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep;
const { groupId } = parseName(rep.name);
if (!buckets.has(groupId)) buckets.set(groupId, []);
buckets.get(groupId).push(g);
}
for (const [, arr] of buckets) {
arr.sort((a, b) => {
const sa = (a.tar ?? a.setup ?? a.dmg ?? a.rep)?.sort ?? 0;
const sb = (b.tar ?? b.setup ?? b.dmg ?? b.rep)?.sort ?? 0;
return sb - sa;
}); });
arr.forEach((g, idx) => { const itemCount = {};
const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; const ext = ".tar.gz";
const base = baseStem(rep.name);
if (idx < 3) { const filteredItems = ret
keepBaseSet.add(base); .filter((item) => item.name.endsWith(ext))
} else { .filter((item) => {
if (g.tar) if (folderName === "nightly") {
[g.tar.key, ...sidecarsFor(g.tar.key)].forEach((k) => const parts = item.name.split("_");
oldKeys.add(k), const groupId = `${parts[0]}_${parts[3]}_${parts[4]}`;
); itemCount[groupId] = itemCount[groupId] || 0;
if (g.setup) if (++itemCount[groupId] <= 3) {
[g.setup.key, ...sidecarsFor(g.setup.key)].forEach((k) => return true;
oldKeys.add(k),
);
if (g.dmg)
[g.dmg.key, ...sidecarsFor(g.dmg.key)].forEach((k) =>
oldKeys.add(k),
);
} }
}); if (!oldItems.find((key) => key === item.key)) {
} // mark tar + sidecars old
} else { oldItems.push(item.key);
for (const g of baseGroups) { oldItems.push(item.key + ".sha256");
const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; oldItems.push(item.key + ".sig");
keepBaseSet.add(baseStem(rep.name));
// windows companion (_setup.exe) + sidecars
if (parts[3] === "windows") {
const setup_key =
item.key.substring(0, item.key.length - ext.length) +
"_setup.exe";
const setup_item = ret.find((i) => i.key == setup_key);
if (setup_item) {
oldItems.push(setup_key);
oldItems.push(setup_key + ".sha256");
oldItems.push(setup_key + ".sig");
} }
} }
// ---------- Build releases: per release (build) we emit three blocks ---------- // darwin companion (.dmg) + sidecars <-- NEW
const releases = new Map(); // releaseId -> { repSort, items:[baseGroup] } if (parts[3] === "darwin") {
for (const g of baseGroups) { const dmg_key =
const rep = g.tar ?? g.setup ?? g.dmg ?? g.rep; item.key.substring(0, item.key.length - ext.length) + ".dmg";
const base = baseStem(rep.name); const dmg_item = ret.find((i) => i.key == dmg_key);
if (!keepBaseSet.has(base)) continue; if (dmg_item) {
oldItems.push(dmg_key);
let r = releases.get(g.releaseId); oldItems.push(dmg_key + ".sha256");
if (!r) { oldItems.push(dmg_key + ".sig");
r = { repSort: rep.sort, items: [] };
releases.set(g.releaseId, r);
} }
r.repSort = Math.max(r.repSort, rep.sort);
r.items.push({ ...g, repSort: rep.sort });
} }
}
// newest releases first return false;
const releaseList = Array.from(releases.entries()).sort( }
(a, b) => b[1].repSort - a[1].repSort, return true;
);
const out = [];
for (const [, bucket] of releaseList) {
// Sort items consistently for all three blocks
const sorted = bucket.items.slice().sort((a, b) => {
const pr = platformRank(a.platform) - platformRank(b.platform);
if (pr !== 0) return pr;
const ar = archRank(a.arch) - archRank(b.arch);
if (ar !== 0) return ar;
return b.repSort - a.repSort; // tie-breaker newest first
}); });
// Block 1: all tar.gz (+ sidecars) const totalCount = filteredItems.length * 3;
for (const g of sorted) { const setup_items = [];
if (!g.tar) continue; const dmg_items = []; // <-- NEW
out.push(g.tar); let setup_offset = 0;
for (const sk of sidecarsFor(g.tar.key)) { let dmg_offset = 0;
const it = byKey[sk];
if (it) out.push(it); for (let i = 0; i < totalCount && i < filteredItems.length; i += 3) {
const parts = filteredItems[i].name.split("_");
// add tar sidecars
let item = ret.filter(
(it) => it.name === filteredItems[i].name + ".sha256",
);
filteredItems.splice(i + 1, 0, ...item);
item = ret.filter((it) => it.name === filteredItems[i].name + ".sig");
filteredItems.splice(i + 2, 0, ...item);
// windows companion block (setup.exe + sidecars)
if (parts[3] === "windows") {
const setup_key =
filteredItems[i].key.substring(
0,
filteredItems[i].key.length - ext.length,
) + "_setup.exe";
const setup_item = ret.find((it) => it.key == setup_key);
if (setup_item) {
const setup_item2 = ret.find((it) => it.key == setup_key + ".sha256");
const setup_item3 = ret.find((it) => it.key == setup_key + ".sig");
setup_items.push([
{ idx: i + 3 + setup_offset },
setup_item,
setup_item2,
setup_item3,
]);
setup_offset += 3;
} }
} }
// Block 2: all _setup.exe (+ sidecars) // darwin companion block (.dmg + sidecars) <-- NEW (mirrors setup flow)
for (const g of sorted) { if (parts[3] === "darwin") {
if (!g.setup) continue; const dmg_key =
out.push(g.setup); filteredItems[i].key.substring(
for (const sk of sidecarsFor(g.setup.key)) { 0,
const it = byKey[sk]; filteredItems[i].key.length - ext.length,
if (it) out.push(it); ) + ".dmg";
}
}
// Block 3: all .dmg (+ sidecars) const dmg_item = ret.find((it) => it.key == dmg_key);
for (const g of sorted) { if (dmg_item) {
if (!g.dmg) continue; const dmg_item2 = ret.find((it) => it.key == dmg_key + ".sha256");
out.push(g.dmg); const dmg_item3 = ret.find((it) => it.key == dmg_key + ".sig");
for (const sk of sidecarsFor(g.dmg.key)) { dmg_items.push([
const it = byKey[sk]; { idx: i + 3 + dmg_offset },
if (it) out.push(it); dmg_item,
dmg_item2,
dmg_item3,
]);
dmg_offset += 3;
} }
} }
} }
return out; // Insert darwin DMG blocks first, then Windows setup blocks (order doesn't matter across platforms)
if (dmg_items.length > 0) {
dmg_items.forEach((items) => {
filteredItems.splice(items[0].idx, 0, items[1]);
filteredItems.splice(items[0].idx + 1, 0, items[2]);
filteredItems.splice(items[0].idx + 2, 0, items[3]);
});
}
if (setup_items.length > 0) {
setup_items.forEach((items) => {
filteredItems.splice(items[0].idx, 0, items[1]);
filteredItems.splice(items[0].idx + 1, 0, items[2]);
filteredItems.splice(items[0].idx + 2, 0, items[3]);
});
}
return filteredItems;
} catch (err) { } catch (err) {
console.error(err); console.error(err);
return [];
} }
return [];
}; };
export { cleanOldItems, createDownloadLink, getBucketFiles }; export { cleanOldItems, createDownloadLink, getBucketFiles };