added dmg
This commit is contained in:
259
src/api.js
259
src/api.js
@@ -7,8 +7,9 @@ import {
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
|
||||
const BUCKET = "repertory";
|
||||
const TAR_EXT = ".tar.gz";
|
||||
|
||||
const oldItems = [];
|
||||
const oldKeys = new Set();
|
||||
|
||||
const s3 = new S3Client({
|
||||
region: "any",
|
||||
@@ -20,27 +21,76 @@ const s3 = new S3Client({
|
||||
},
|
||||
});
|
||||
|
||||
/** Helpers **/
|
||||
|
||||
const toListed = (folderKey) => (o) => {
|
||||
const d = o.LastModified ?? new Date(0);
|
||||
const date = d
|
||||
.toLocaleString("en-US", {
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
hour12: false,
|
||||
})
|
||||
.replace(/,/g, "");
|
||||
return {
|
||||
date,
|
||||
sort: d.getTime(),
|
||||
name: (o.Key ?? "").replace(folderKey, ""),
|
||||
key: o.Key ?? "",
|
||||
};
|
||||
};
|
||||
|
||||
const byNewest = (a, b) => b.sort - a.sort;
|
||||
|
||||
const sidecarsFor = (key) => [key + ".sha256", key + ".sig"];
|
||||
|
||||
const parseName = (fname) => {
|
||||
// Expected pattern: parts[3] = platform (windows|darwin|linux), parts[4] = arch
|
||||
const parts = fname.split("_");
|
||||
const platform = parts[3] ?? "";
|
||||
const arch = parts[4] ?? "";
|
||||
const groupId = `${parts[0] ?? ""}_${platform}_${arch}`;
|
||||
return { parts, platform, arch, groupId };
|
||||
};
|
||||
|
||||
const windowsCompanionFromTar = (tarKey) => {
|
||||
// .../name.tar.gz -> .../name_setup.exe
|
||||
const base = tarKey.slice(0, -TAR_EXT.length);
|
||||
return `${base}_setup.exe`;
|
||||
};
|
||||
|
||||
const darwinCompanionFromTar = (tarKey) => {
|
||||
// .../name.tar.gz -> .../name.dmg
|
||||
const base = tarKey.slice(0, -TAR_EXT.length);
|
||||
return `${base}.dmg`;
|
||||
};
|
||||
|
||||
/** Public API **/
|
||||
|
||||
const cleanOldItems = async () => {
|
||||
console.log(`cleaning|count|${oldItems.length}`);
|
||||
while (oldItems.length > 0) {
|
||||
try {
|
||||
const key = oldItems.pop();
|
||||
console.log(`cleaning|key|${key}`);
|
||||
await s3.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: BUCKET,
|
||||
Key: key,
|
||||
}),
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
const keys = Array.from(oldKeys);
|
||||
console.log(`cleaning|count|${keys.length}`);
|
||||
if (keys.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const deletes = keys.map((Key) =>
|
||||
s3
|
||||
.send(new DeleteObjectCommand({ Bucket: BUCKET, Key }))
|
||||
.then(() => console.log(`cleaning|key|${Key}`))
|
||||
.catch((err) => console.error(`cleaning|error|${Key}`, err)),
|
||||
);
|
||||
|
||||
await Promise.allSettled(deletes);
|
||||
oldKeys.clear();
|
||||
};
|
||||
|
||||
const createDownloadLink = async (key) => {
|
||||
let filename = key.split("/");
|
||||
filename = filename[filename.length - 1];
|
||||
const filename = key.split("/").pop() ?? "download.bin";
|
||||
return await getSignedUrl(
|
||||
s3,
|
||||
new GetObjectCommand({
|
||||
@@ -54,121 +104,106 @@ const createDownloadLink = async (key) => {
|
||||
|
||||
const getBucketFiles = async (folderName) => {
|
||||
try {
|
||||
folderName = folderName.toLowerCase();
|
||||
const folderKey = encodeURIComponent(folderName) + "/";
|
||||
const folder = (folderName ?? "").toLowerCase();
|
||||
const folderKey = encodeURIComponent(folder) + "/";
|
||||
|
||||
const data = await s3.send(
|
||||
new ListObjectsCommand({
|
||||
Bucket: BUCKET,
|
||||
Prefix: folderKey,
|
||||
}),
|
||||
);
|
||||
const ret = data.Contents.filter((item) => item.Key !== folderKey)
|
||||
.map((item) => {
|
||||
return {
|
||||
date: item.LastModified.toLocaleDateString("en-US", {
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
}).replace(/,/g, ""),
|
||||
sort: item.LastModified.getTime(),
|
||||
name: item.Key.replace(folderKey, ""),
|
||||
key: item.Key,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => {
|
||||
return a.sort > b.sort ? -1 : a.sort < b.sort ? 1 : 0;
|
||||
});
|
||||
const itemCount = {};
|
||||
const ext = ".tar.gz";
|
||||
|
||||
const filteredItems = ret
|
||||
.filter((item) => item.name.endsWith(ext))
|
||||
.filter((item) => {
|
||||
if (folderName === "nightly") {
|
||||
const parts = item.name.split("_");
|
||||
const groupId = `${parts[0]}_${parts[3]}_${parts[4]}`;
|
||||
itemCount[groupId] = itemCount[groupId] || 0;
|
||||
if (++itemCount[groupId] <= 3) {
|
||||
return true;
|
||||
const contents = (data.Contents ?? []).filter((it) => it.Key !== folderKey);
|
||||
|
||||
// Normalize listing
|
||||
const listed = contents.map(toListed(folderKey)).sort(byNewest);
|
||||
|
||||
// Consider only .tar.gz for retention policy
|
||||
const tarItems = listed.filter((i) => i.name.endsWith(TAR_EXT));
|
||||
|
||||
// Nightly policy: keep top 3 per (name_platform_arch); mark others + companions for deletion
|
||||
const keepSet = new Set();
|
||||
if (folder === "nightly") {
|
||||
const groupCounts = {};
|
||||
for (const t of tarItems) {
|
||||
const { groupId, platform } = parseName(t.name);
|
||||
groupCounts[groupId] = (groupCounts[groupId] ?? 0) + 1;
|
||||
const count = groupCounts[groupId];
|
||||
|
||||
if (count <= 3) {
|
||||
keepSet.add(t.key);
|
||||
} else {
|
||||
[t.key, ...sidecarsFor(t.key)].forEach((k) => oldKeys.add(k));
|
||||
|
||||
if (platform === "windows") {
|
||||
const setupKey = windowsCompanionFromTar(t.key);
|
||||
[setupKey, ...sidecarsFor(setupKey)].forEach((k) => oldKeys.add(k));
|
||||
} else if (platform === "darwin") {
|
||||
const dmgKey = darwinCompanionFromTar(t.key);
|
||||
[dmgKey, ...sidecarsFor(dmgKey)].forEach((k) => oldKeys.add(k));
|
||||
}
|
||||
if (!oldItems.find((key) => key === item.key)) {
|
||||
oldItems.push(item.key);
|
||||
oldItems.push(item.key + ".sha256");
|
||||
oldItems.push(item.key + ".sig");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tarItems.forEach((t) => keepSet.add(t.key));
|
||||
}
|
||||
|
||||
if (parts[3] === "windows") {
|
||||
const setup_key =
|
||||
item.key.substring(0, item.key.length - ext.length) +
|
||||
"_setup.exe";
|
||||
// Build final output list (tar + sidecars + optional platform companion + its sidecars)
|
||||
const byKey = Object.fromEntries(listed.map((i) => [i.key, i]));
|
||||
const out = [];
|
||||
|
||||
const setup_item = ret.find((item) => item.key == setup_key);
|
||||
if (setup_item) {
|
||||
oldItems.push(setup_key);
|
||||
oldItems.push(setup_key + ".sha256");
|
||||
oldItems.push(setup_key + ".sig");
|
||||
}
|
||||
for (const t of tarItems) {
|
||||
if (!keepSet.has(t.key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// tar.gz
|
||||
out.push(t);
|
||||
|
||||
// tar sidecars
|
||||
for (const sk of sidecarsFor(t.key)) {
|
||||
const it = byKey[sk];
|
||||
if (it) {
|
||||
out.push(it);
|
||||
}
|
||||
}
|
||||
|
||||
// platform companions
|
||||
const { platform } = parseName(t.name);
|
||||
|
||||
if (platform === "windows") {
|
||||
const setupKey = windowsCompanionFromTar(t.key);
|
||||
const setupItem = byKey[setupKey];
|
||||
if (setupItem) {
|
||||
out.push(setupItem);
|
||||
for (const sk of sidecarsFor(setupKey)) {
|
||||
const it = byKey[sk];
|
||||
if (it) {
|
||||
out.push(it);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
const totalCount = filteredItems.length * 3;
|
||||
const setup_items = [];
|
||||
let setup_offset = 0;
|
||||
for (let i = 0; i < totalCount && i < filteredItems.length; i += 3) {
|
||||
const parts = filteredItems[i].name.split("_");
|
||||
let item = ret.filter(
|
||||
(item) => item.name === filteredItems[i].name + ".sha256",
|
||||
);
|
||||
filteredItems.splice(i + 1, 0, ...item);
|
||||
item = ret.filter((item) => item.name === filteredItems[i].name + ".sig");
|
||||
filteredItems.splice(i + 2, 0, ...item);
|
||||
|
||||
if (parts[3] === "windows") {
|
||||
const setup_key =
|
||||
filteredItems[i].key.substring(
|
||||
0,
|
||||
filteredItems[i].key.length - ext.length,
|
||||
) + "_setup.exe";
|
||||
|
||||
const setup_item = ret.find((item) => item.key == setup_key);
|
||||
if (setup_item) {
|
||||
const setup_item2 = ret.find(
|
||||
(item) => item.key == setup_key + ".sha256",
|
||||
);
|
||||
const setup_item3 = ret.find(
|
||||
(item) => item.key == setup_key + ".sig",
|
||||
);
|
||||
setup_items.push([
|
||||
{ idx: i + 3 + setup_offset },
|
||||
setup_item,
|
||||
setup_item2,
|
||||
setup_item3,
|
||||
]);
|
||||
setup_offset += 3;
|
||||
} else if (platform === "darwin") {
|
||||
const dmgKey = darwinCompanionFromTar(t.key);
|
||||
const dmgItem = byKey[dmgKey];
|
||||
if (dmgItem) {
|
||||
out.push(dmgItem);
|
||||
for (const sk of sidecarsFor(dmgKey)) {
|
||||
const it = byKey[sk];
|
||||
if (it) {
|
||||
out.push(it);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (setup_items.length > 0) {
|
||||
setup_items.forEach((items) => {
|
||||
filteredItems.splice(items[0].idx, 0, items[1]);
|
||||
filteredItems.splice(items[0].idx + 1, 0, items[2]);
|
||||
filteredItems.splice(items[0].idx + 2, 0, items[3]);
|
||||
});
|
||||
}
|
||||
|
||||
return filteredItems;
|
||||
return out;
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return [];
|
||||
}
|
||||
|
||||
return [];
|
||||
};
|
||||
|
||||
export { cleanOldItems, createDownloadLink, getBucketFiles };
|
||||
|
||||
Reference in New Issue
Block a user