From da8d0fcc7edf20749f374637bce736353b883c0f Mon Sep 17 00:00:00 2001 From: Cory Sanin Date: Sun, 12 Jan 2025 23:11:58 -0500 Subject: [PATCH] Create BuildController --- src/BuildController.ts | 156 +++++++++++++++++++++++++++++++++++++++++ src/DB.ts | 54 +++++++++++--- src/Web.ts | 14 +++- src/index.ts | 4 ++ styles/01-styles.scss | 36 +++++++++- views/build.ejs | 5 +- 6 files changed, 254 insertions(+), 15 deletions(-) create mode 100644 src/BuildController.ts diff --git a/src/BuildController.ts b/src/BuildController.ts new file mode 100644 index 0000000..975cec6 --- /dev/null +++ b/src/BuildController.ts @@ -0,0 +1,156 @@ +import spawn from 'child_process'; +import EventEmitter from 'events'; +import type { Build, DB } from "./DB.ts"; + +const docker_images = { + arch: 'corysanin/archy-build-thing:arch', + artix: 'corysanin/archy-build-thing:artix', +} + +type LogType = 'std' | 'err' | 'finish'; + +interface BuildEvent { + id: number; + type: LogType; + message: any; +} + +class BuildController extends EventEmitter { + private db: DB; + private process: spawn.ChildProcess | null = null; + private running: boolean = false; + + constructor(config = {}) { + super(); + setInterval(this.triggerBuild, 60000); + } + + triggerBuild = () => { + if (!this.running) { + this.running = true; + this.kickOffBuild(); + return true; + } + return false; + } + + private kickOffBuild = async () => { + this.running = true; + const build = await this.db.dequeue(); + if (build === null) { + this.running = false; + return; + } + try { + await this.pullImage(build.distro); + await this.build(build); + } + catch (e) { + console.error(e); + this.db.finishBuild(build.id, 'error'); + } + this.kickOffBuild(); + } + + private pullImage = (distro: string) => { + return new Promise((resolve, reject) => { + if (!(distro in docker_images)) { + return reject(); + } + const docker = spawn.spawn('docker', ['pull', docker_images[distro]]); + docker.stdout.on('data', (data) => { + console.log(`${data}`); + }); + docker.stderr.on('data', (data) => { + console.error(`${data}`); + }); + docker.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(code); + } + }); + docker.on('error', (err) => { + reject(err); + }); + }); + } + + private build = async (build: Build) => { + return new Promise((resolve, reject) => { + const docker = this.process = spawn.spawn('docker', this.createBuildParams(build)); + docker.on('spawn', () => { + const remainder = { + std: '', + err: '' + } + this.db.startBuild(build.id, docker.pid); + + let createLogFunction = (type: LogType) => { + return (data: Buffer | string) => { + const str = data.toString(); + const readyToLog = remainder[type] + str.substring(0, str.lastIndexOf('\n')); + remainder[type] = str.substring(str.lastIndexOf('\n')); + this.db.appendLog(build.id, readyToLog); + this.emitLog({ + id: build.id, + type: type, + message: readyToLog + }); + }; + }; + + docker.stdout.on('data', createLogFunction('std')); + docker.stderr.on('data', createLogFunction('err')); + }); + docker.on('close', (code) => { + this.process = null; + this.emitLog({ + id: build.id, + type: 'finish', + message: code + }); + + if (code === 0) { + this.db.finishBuild(build.id, 'success'); + resolve(); + } + else { + this.db.finishBuild(build.id, 'error'); + reject(code); + } + }); + }); + } + + private emitLog = (msg: BuildEvent) => { + this.emit('log', msg); + } + + private createBuildParams = (build: Build) => { + // TODO: implement patch + const params = ['run', '--rm', '-e', `REPO=${build.repo}`]; + if (build.commit) { + // TODO: implement COMMIT + params.push('-e', `COMMIT=${build.commit}`); + } + params.push(docker_images[build.distro]); + return params; + } + + cancelBuild = async (pid: number) => { + const p = this.process + if (p && p.pid === pid) { + return p.kill(); + } + } + + setDB = (db: DB) => { + this.db = db; + } +} + +export default BuildController; +export { BuildController }; +export type { }; \ No newline at end of file diff --git a/src/DB.ts b/src/DB.ts index 2eabeec..73fce8a 100644 --- a/src/DB.ts +++ b/src/DB.ts @@ -1,6 +1,9 @@ import { Sequelize, DataTypes, Op, } from 'sequelize'; import type { ModelStatic, Filterable } from 'sequelize'; +type Status = 'queued' | 'running' | 'cancelled' | 'success' | 'error'; +type Dependencies = 'stable' | 'testing' | 'staging'; + interface DBConfig { db?: string; user?: string; @@ -9,6 +12,20 @@ interface DBConfig { port?: number; } +interface Build { + id: number; + repo: string; + commit?: string; + patch?: string; + distro: string; + dependencies: Dependencies; + startTime?: Date; + endTime?: Date; + status: Status; + pid?: number; + log?: string; +} + const MONTH = 1000 * 60 * 60 * 24 * 24; const FRESH = { [Op.or]: [ @@ -16,7 +33,7 @@ const FRESH = { { startTime: { [Op.is]: null } } ] } -const SELECT = ['id', 'repo', 'commit', 'distro', 'startTime', 'endTime', 'status']; +const SELECT = ['id', 'repo', 'commit', 'distro', 'dependencies', 'startTime', 'endTime', 'status']; class DB { private build: ModelStatic; @@ -50,6 +67,11 @@ class DB { type: DataTypes.STRING, allowNull: false }, + dependencies: { + type: DataTypes.ENUM('stable', 'testing', 'staging'), + allowNull: false, + defaultValue: 'stable' + }, startTime: { type: DataTypes.DATE, allowNull: true @@ -99,7 +121,7 @@ class DB { }); } - public async finishBuild(id: number, status: string): Promise { + public async finishBuild(id: number, status: Status): Promise { await this.build.update({ endTime: new Date(), status @@ -111,8 +133,9 @@ class DB { } public async appendLog(id: number, log: string): Promise { + const sanitizedLog = log.replace(/'/g, "''"); await this.build.update({ - log: Sequelize.literal(`log || '${log}'`) + log: Sequelize.literal(`log || '${sanitizedLog}'`) }, { where: { id @@ -120,11 +143,11 @@ class DB { }); } - public async getBuild(id: number): Promise { + public async getBuild(id: number): Promise { return await this.build.findByPk(id); } - public async getBuilds(): Promise { + public async getBuilds(): Promise { return await this.build.findAll({ attributes: SELECT, order: [['id', 'DESC']], @@ -132,7 +155,7 @@ class DB { }); } - public async getBuildsByStatus(status: string): Promise { + public async getBuildsByStatus(status: Status): Promise { return await this.build.findAll({ attributes: SELECT, order: [['id', 'DESC']], @@ -143,7 +166,7 @@ class DB { }); } - public async getBuildsByDistro(distro: string): Promise { + public async getBuildsByDistro(distro: string): Promise { return await this.build.findAll({ attributes: SELECT, order: [['id', 'DESC']], @@ -154,7 +177,7 @@ class DB { }); } - public async getBuildsBy(filterable: Filterable): Promise { + public async getBuildsBy(filterable: Filterable): Promise { return await this.build.findAll({ attributes: SELECT, order: [['id', 'DESC']], @@ -165,7 +188,18 @@ class DB { }); } - public async searchBuilds(query: string): Promise { + public async dequeue(): Promise { + return await this.build.findOne({ + attributes: SELECT, + order: [['id', 'ASC']], + where: { + status: 'queued' + }, + limit: 1 + }); + } + + public async searchBuilds(query: string): Promise { return await this.build.findAll({ attributes: SELECT, order: [['id', 'DESC']], @@ -193,4 +227,4 @@ class DB { export default DB; export { DB }; -export type { DBConfig }; +export type { DBConfig, Status, Build }; diff --git a/src/Web.ts b/src/Web.ts index 01707f2..f022e1f 100644 --- a/src/Web.ts +++ b/src/Web.ts @@ -4,6 +4,7 @@ import type { Express } from "express"; import express, { application } from 'express'; import bodyParser from "body-parser"; import type { DB } from "./DB.ts"; +import type { BuildController } from "./BuildController.ts"; interface WebConfig { port?: number; @@ -19,6 +20,7 @@ function notStupidParseInt(v: string | undefined): number { class Web { private _webserver: http.Server | null = null; private db: DB; + private buildController: BuildController; constructor(options: WebConfig = {}) { const app: Express = express(); @@ -71,8 +73,9 @@ class Web { }); app.post('/build/?', async (req, res) => { - const build = await this.db.createBuild(req.body.repo, req.body.commit || null, req.body.patch || null, req.body.distro); - res.redirect(`/build/${build}`); + const buildId = await this.db.createBuild(req.body.repo, req.body.commit || null, req.body.patch || null, req.body.distro); + res.redirect(`/build/${buildId}`); + this.buildController.triggerBuild(); }); app.get('/build/:num/?', async (req, res) => { @@ -87,7 +90,8 @@ class Web { titlesuffix: `Build #${req.params.num}`, description: `Building ${build.repo} on ${build.distro}` }, - build + build, + log: build.log?.split('\n') }); }); @@ -116,6 +120,10 @@ class Web { setDB = (db: DB) => { this.db = db; } + + setBuildController = (buildController: BuildController) => { + this.buildController = buildController; + } } export default Web; diff --git a/src/index.ts b/src/index.ts index 4d5d59e..629b35c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import { Web } from './Web.ts'; import { DB } from './DB.ts'; +import { BuildController } from './BuildController.ts'; import type { WebConfig } from './Web.ts'; import type { DBConfig } from './DB.ts'; @@ -13,9 +14,12 @@ interface compositeConfig { const config: compositeConfig = JSON.parse(await fs.promises.readFile(process.env.config || path.join('config', 'config.json'), 'utf-8')); const web = new Web(config.web); +const buildController = new BuildController(); await new Promise((resolve) => setTimeout(resolve, 1500)); const db = new DB(config.db); web.setDB(db); +web.setBuildController(buildController); +buildController.setDB(db); process.on('SIGTERM', () => { web.close(); diff --git a/styles/01-styles.scss b/styles/01-styles.scss index b883859..5308587 100644 --- a/styles/01-styles.scss +++ b/styles/01-styles.scss @@ -176,4 +176,38 @@ input[type="submit"] { .span-2 { grid-column: span 2; } -} \ No newline at end of file +} + +.logs { + font-family: 'Courier New', monospace; + border-left: #d96a14 .2em solid; + background: #2a2a2a; + margin-top: 1em; + overflow-x: auto; + overflow-wrap: anywhere; + + p { + padding: .1em .5em; + min-height: 1.5em; + + &:nth-child(even) { + background-color: #222; + } + } +} + +.sidebar_search { + padding: .3em 1em; + + form input { + font-size: 1.1em; + width: 100%; + background: #222; + color: var(--color-text); + border: var(--primary-light) solid .12em; + + &:hover { + background: #2a2a2a; + } + } +} diff --git a/views/build.ejs b/views/build.ejs index 3e05c47..839012c 100644 --- a/views/build.ejs +++ b/views/build.ejs @@ -18,7 +18,10 @@ <%= build.startTime %>

Full logs

-
+
+ <% (log || []).forEach(line => { %> +

<%= line %>

+ <% }) %>