diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000..ceb3cfe --- /dev/null +++ b/.babelrc @@ -0,0 +1,13 @@ +{ + "presets": [ + ["@babel/env", { + "targets": { + "node": "current" + } + }] + ], + "plugins": [ + "@babel/plugin-proposal-class-properties", + "@babel/plugin-proposal-object-rest-spread" + ] +} diff --git a/.cspell/words.txt b/.cspell/words.txt new file mode 100644 index 0000000..b3ac530 --- /dev/null +++ b/.cspell/words.txt @@ -0,0 +1 @@ +fifthgrid \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2d3ba5b --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.git/ +dist/ +node_modules/ +package-lock.json +bin/ +run.sh diff --git a/.jenkins_builds b/.jenkins_builds new file mode 100644 index 0000000..337eec3 --- /dev/null +++ b/.jenkins_builds @@ -0,0 +1,20 @@ +#!groovy + +pipeline { + agent any + + options { + disableConcurrentBuilds() + } + + stages { + stage('build') { + steps { + nodejs(nodeJSInstallationName: 'current') { + sh 'npm i' + sh 'npm run pkg' + } + } + } + } +} diff --git a/.vim/coc-settings.json b/.vim/coc-settings.json new file mode 100644 index 0000000..c4c21aa --- /dev/null +++ b/.vim/coc-settings.json @@ -0,0 +1,5 @@ +{ + "cSpell.words": [ + "presigner" + ] +} \ No newline at end of file diff --git a/LICENSE b/LICENSE index 0d21b9e..0c7b8f2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 Public +Copyright (c) <2022> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/README.md b/README.md index 27f1885..cd77826 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # fifthgrid_browser -Server to host `fifthgrid` application binaries \ No newline at end of file +Server to host `fifthgrid` application binaries diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..cba22db --- /dev/null +++ b/cspell.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", + "version": "0.2", + "dictionaries": ["workspace-words", "user-words"], + "dictionaryDefinitions": [{ + "name": "workspace-words", + "path": "./.cspell/words.txt", + "addWords": true + }, + { + "name": "user-words", + "path": "C:\\.desktop\\.cspell\\user_words.txt", + "addWords": true + } + ] +} \ No newline at end of file diff --git a/developer.pub b/developer.pub new file mode 100644 index 0000000..2075fac --- /dev/null +++ b/developer.pub @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqXedleDOugdk9sBpgFOA +0+MogIbBF7+iXIIHv8CRBbrrf8nxLSgQvbHQIP0EklebDgLZRgyGI3SSQYj7D957 +uNf1//dpkELNzfuezgAyFer9+iH4Svq46HADp5k+ugaK0mMDZM7OLOgo7415/+z4 +NIQopv8prMFdxkShr4e4dpR+S6LYMYMVjsi1gnYWaZJMWgzeZouXFSscS1/XDXSE +vr1Jfqme+RmB4Q2QqGcDrY2ijumCJYJzQqlwG6liJ4FNg0U3POTCQDhQmuUoEJe0 +/dyiWlo48WQbBu6gUDHbTCCUSZPs2Lc9l65MqOCpX76+VXPYetZgqpMF4GVzb2y9 +kETxFNpiMYBlOBZk0I1G33wqVmw46MI5IZMQ2z2F8Mzt1hByUNTgup2IQELCv1a5 +a2ACs2TBRuAy1REeHhjLgiA/MpoGX7TpoHCGyo8jBChJVpP9ZHltKoChwDC+bIyx +rgYH3jYDkl2FFuAUJ8zAZl8U1kjqZb9HGq9ootMk34Dbo3IVkc2azB2orEP9F8QV +KxvZZDA9FAFEthSiNf5soJ6mZGLi0es5EWPoKMUEd9tG5bP980DySAWSSRK0AOfE +QShT/z7oG79Orxyomwrb8ZJCi7wEfcCuK1NWgqLVUgXhpi2J9WYS6DAbF3Oh3Hhl +DYSHlcfFBteqNDlR2uFInIECAwEAAQ== +-----END PUBLIC KEY----- diff --git a/package.json b/package.json new file mode 100644 index 0000000..d4413b5 --- /dev/null +++ b/package.json @@ -0,0 +1,47 @@ +{ + "name": "fifthgrid-browser", + "version": "0.0.1", + "private": true, + "bin": "dist/bin/www", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "build": "rimraf dist && babel src --out-dir dist --copy-files", + "pkg": "npm run build && pkg --out-path bin/ .", + "start": "npm run build && node dist/bin/www", + "start:dev": "cross-env NODE_ENV=development nodemon --exec ./node_modules/.bin/babel-node src/bin/www" + }, + "dependencies": { + "@aws-sdk/client-cognito-identity": "^3.53.0", + "@aws-sdk/client-s3": "^3.53.1", + "@aws-sdk/s3-request-presigner": "^3.53.1", + "cookie-parser": "~1.4.4", + "debug": "~2.6.9", + "express": "~4.16.1", + "http-errors": "~1.6.3", + "morgan": "~1.9.1", + "pug": "3.0.2", + "request": "^2.88.2" + }, + "devDependencies": { + "@babel/cli": "^7.17.6", + "@babel/core": "^7.17.5", + "@babel/node": "^7.16.8", + "@babel/plugin-proposal-class-properties": "^7.16.7", + "@babel/plugin-proposal-object-rest-spread": "^7.17.3", + "@babel/preset-env": "^7.16.11", + "cross-env": "^7.0.3", + "nodemon": "^2.0.15", + "pkg": "^5.8.1", + "rimraf": "^3.0.2", + "semgrep": "^0.0.1" + }, + "pkg": { + "assets": [ + "node_modules/**/*", + "dist/**/*" + ], + "targets": [ + "node18-linux" + ] + } +} diff --git a/src/api.js b/src/api.js new file mode 100644 index 0000000..b2cf154 --- /dev/null +++ b/src/api.js @@ -0,0 +1,127 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + ListObjectsCommand, + S3Client, +} from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; + +const BUCKET = "repertory"; + +const oldItems = []; + +const s3 = new S3Client({ + region: "any", + endpoint: "https://gateway.storjshare.io", + forcePathStyle: true, + credentials: { + accessKeyId: process.env.R_AWS_KEY, + secretAccessKey: process.env.R_AWS_SECRET, + }, +}); + +const cleanOldItems = async () => { + console.log("cleanOldItems", oldItems.length); + while (oldItems.length > 0) { + const key = oldItems.pop(); + await s3.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: key, + }), + ); + console.log(key); + } +}; + +const createDownloadLink = async (key) => { + let filename = key.split("/"); + filename = filename[filename.length - 1]; + return await getSignedUrl( + s3, + new GetObjectCommand({ + Bucket: BUCKET, + Key: key, + ResponseContentDisposition: `attachment; filename="${filename}"`, + }), + { expiresIn: 3600 }, + ); +}; + +const getAllFiles = async () => { + const data = await s3.send( + new ListObjectsCommand({ + Bucket: BUCKET, + Delimiter: "/", + }), + ); + console.log(data); +}; + +const getBucketFiles = async (folderName) => { + try { + folderName = folderName.toLowerCase(); + const folderKey = encodeURIComponent(folderName) + "/"; + const data = await s3.send( + new ListObjectsCommand({ + Bucket: BUCKET, + Prefix: folderKey, + }), + ); + const ret = data.Contents.filter((item) => item.Key !== folderKey) + .map((item) => { + return { + date: item.LastModified.toLocaleDateString("en-US", { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }).replace(/,/g, ""), + sort: item.LastModified.getTime(), + name: item.Key.replace(folderKey, ""), + key: item.Key, + }; + }) + .sort((a, b) => { + return a.sort > b.sort ? -1 : a.sort < b.sort ? 1 : 0; + }); + const itemCount = {}; + const filteredItems = ret + .filter((item) => item.name.endsWith(".tar.gz")) + .filter((item) => { + if (folderName === "nightly") { + const parts = item.name.split("_"); + const groupId = `${parts[0]}_${parts[3]}_${parts[4]}`; + itemCount[groupId] = itemCount[groupId] || 0; + if (++itemCount[groupId] <= 3) { + return true; + } + if (!oldItems.find((key) => key === item.key)) { + oldItems.push(item.key); + oldItems.push(item.key + ".sha256"); + oldItems.push(item.key + ".sig"); + } + return false; + } + return true; + }); + const totalCount = filteredItems.length * 3; + for (let i = 0; i < totalCount && i < filteredItems.length; i += 3) { + let item = ret.filter( + (item) => item.name === filteredItems[i].name + ".sha256", + ); + filteredItems.splice(i + 1, 0, ...item); + item = ret.filter((item) => item.name === filteredItems[i].name + ".sig"); + filteredItems.splice(i + 2, 0, ...item); + } + return filteredItems; + } catch (err) { + console.log(err); + } + + return []; +}; + +export { cleanOldItems, createDownloadLink, getBucketFiles }; diff --git a/src/app.js b/src/app.js new file mode 100644 index 0000000..99d73ab --- /dev/null +++ b/src/app.js @@ -0,0 +1,49 @@ +import createHttpError from "http-errors"; +import express from "express"; +import path from "path"; +import cookieParser from "cookie-parser"; +import logger from "morgan"; + +import indexRouter from "./routes/index"; + +if (!process.env.R_AWS_KEY) { + console.log("FATAL: 'R_AWS_KEY' environment variable is not set"); + process.exit(-1); +} + +if (!process.env.R_AWS_SECRET) { + console.log("FATAL: 'R_AWS_SECRET' environment variable is not set"); + process.exit(-2); +} + +const app = express().set("env", process.env.NODE_ENV || "production"); + +const errorHandler = (err, res) => { + res.locals.message = err.message || "unknown error"; + res.locals.error = app.get("env") === "development" ? err : {}; + res.render("errors", { + message: "An error occurred", + error: { status: err.status || 500 }, + }); +}; + +app.set("views", path.join(__dirname, "views")); +app.set("view engine", "pug"); + +app.use(logger("dev")); +app.use(express.json()); +app.use(express.urlencoded({ extended: false })); +app.use(cookieParser()); +app.use(express.static(path.join(__dirname, "public"))); + +app.use("/", indexRouter); + +app.use((err, req, res, next) => { + errorHandler(err, res); +}); + +app.use((_, res) => { + errorHandler(createHttpError(404), res); +}); + +module.exports = app; diff --git a/src/mixins/download.pug b/src/mixins/download.pug new file mode 100644 index 0000000..c7beb63 --- /dev/null +++ b/src/mixins/download.pug @@ -0,0 +1,6 @@ +mixin download(file) + .download + |[#{file.date}] + | + a(href='/'+`download?key=${file.key}`) #{file.name} + diff --git a/src/mixins/stage.pug b/src/mixins/stage.pug new file mode 100644 index 0000000..006f862 --- /dev/null +++ b/src/mixins/stage.pug @@ -0,0 +1,3 @@ +mixin stage(name) + .stage + a(href='/'+`stage?name=${name}`) #{name} diff --git a/src/public/stylesheets/style.css b/src/public/stylesheets/style.css new file mode 100644 index 0000000..ebb19dd --- /dev/null +++ b/src/public/stylesheets/style.css @@ -0,0 +1,29 @@ +body { + padding: 0; + margin: 10px; + font: 16px "Lucida Grande", Helvetica, Arial, sans-serif; +} + +a { + display: inline-block; + color: #00B7FF; + font-size: 18px; + margin-bottom: 8px; +} + +.download { + width: 100%; +} + +.stage { + text-transform: capitalize; +} + +.public_key > textarea { + height: 415px; + width: 100%; + border: none; + resize: none; + font-family: monospace; + outline: 0; +} diff --git a/src/routes/index.js b/src/routes/index.js new file mode 100644 index 0000000..96daced --- /dev/null +++ b/src/routes/index.js @@ -0,0 +1,82 @@ +import express from "express"; +import { cleanOldItems, createDownloadLink, getBucketFiles } from "../api"; + +const PUBLIC_KEY = `-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqXedleDOugdk9sBpgFOA +0+MogIbBF7+iXIIHv8CRBbrrf8nxLSgQvbHQIP0EklebDgLZRgyGI3SSQYj7D957 +uNf1//dpkELNzfuezgAyFer9+iH4Svq46HADp5k+ugaK0mMDZM7OLOgo7415/+z4 +NIQopv8prMFdxkShr4e4dpR+S6LYMYMVjsi1gnYWaZJMWgzeZouXFSscS1/XDXSE +vr1Jfqme+RmB4Q2QqGcDrY2ijumCJYJzQqlwG6liJ4FNg0U3POTCQDhQmuUoEJe0 +/dyiWlo48WQbBu6gUDHbTCCUSZPs2Lc9l65MqOCpX76+VXPYetZgqpMF4GVzb2y9 +kETxFNpiMYBlOBZk0I1G33wqVmw46MI5IZMQ2z2F8Mzt1hByUNTgup2IQELCv1a5 +a2ACs2TBRuAy1REeHhjLgiA/MpoGX7TpoHCGyo8jBChJVpP9ZHltKoChwDC+bIyx +rgYH3jYDkl2FFuAUJ8zAZl8U1kjqZb9HGq9ootMk34Dbo3IVkc2azB2orEP9F8QV +KxvZZDA9FAFEthSiNf5soJ6mZGLi0es5EWPoKMUEd9tG5bP980DySAWSSRK0AOfE +QShT/z7oG79Orxyomwrb8ZJCi7wEfcCuK1NWgqLVUgXhpi2J9WYS6DAbF3Oh3Hhl +DYSHlcfFBteqNDlR2uFInIECAwEAAQ== +-----END PUBLIC KEY-----`; + +const router = express.Router(); +const stages = ["release", "nightly", "alpha", "beta", "RC"]; + +router.get("/", (_, res, next) => { + try { + res.render("index", { + title: "Fifthgrid Builds", + stages, + }); + } catch (err) { + next(err); + } +}); + +router.get("/download", async (req, res, next) => { + try { + res.redirect(await createDownloadLink(req.query.key.toString())); + } catch (err) { + next(err); + } +}); + +router.get("/stage", async (req, res, next) => { + try { + const name = req.query.name.toString(); + if (!stages.includes(name)) { + res.render("errors", { + message: "An error occurred", + error: { status: 404 }, + }); + return; + } + + const files = await getBucketFiles(name); + res.render("stage", { + title: "Fifthgrid Builds", + name: name, + public_key: PUBLIC_KEY, + files: files, + }); + } catch (err) { + next(err); + } +}); + +setInterval( + async () => { + try { + for (const stage of stages) { + try { + await getBucketFiles(stage); + } catch (e) { + console.log(e); + } + } + await cleanOldItems(); + } catch (e) { + console.log(e); + } + }, + 15 * 60 * 1000, +); + +export default router; diff --git a/src/views/errors.pug b/src/views/errors.pug new file mode 100644 index 0000000..51ec12c --- /dev/null +++ b/src/views/errors.pug @@ -0,0 +1,6 @@ +extends layout + +block content + h1= message + h2= error.status + pre #{error.stack} diff --git a/src/views/index.pug b/src/views/index.pug new file mode 100644 index 0000000..15dbe88 --- /dev/null +++ b/src/views/index.pug @@ -0,0 +1,9 @@ +extends layout + +include ../mixins/stage.pug + +block content + h1= title + h3= "Currently only 'nightly' and 'rc' builds are available" + each name in stages + +stage(name) diff --git a/src/views/layout.pug b/src/views/layout.pug new file mode 100644 index 0000000..15af079 --- /dev/null +++ b/src/views/layout.pug @@ -0,0 +1,7 @@ +doctype html +html + head + title= title + link(rel='stylesheet', href='/stylesheets/style.css') + body + block content diff --git a/src/views/stage.pug b/src/views/stage.pug new file mode 100644 index 0000000..10b9d48 --- /dev/null +++ b/src/views/stage.pug @@ -0,0 +1,15 @@ +extends layout + +include ../mixins/download.pug + +block content + h1= title + ' [' + name + ']' + h2= 'Public Key' + .public_key + textarea(readonly) #{public_key} + h2= 'Available Downloads' + each file, idx in files + if idx % 3 == 0 + hr + +download(file) + hr