mirror of
https://github.com/alkimake/paperclip.git
synced 2026-06-16 02:40:39 +09:00
[codex] Add issue monitor liveness controls (#4988)
## Thinking Path > - Paperclip is a control plane for autonomous AI companies where work must stay observable, governable, and recoverable. > - The task/heartbeat subsystem owns agent execution continuity, issue state transitions, and visible recovery behavior. > - Waiting on an external service is not the same as being blocked when the assignee still owns a future check. > - The gap was that agents had no first-class one-shot monitor state for external-service waits, so recovery could look stalled or require ad hoc comments. > - This pull request adds bounded issue monitors that can wake the owner, clear exhausted waits, and produce explicit recovery behavior. > - It also surfaces monitor status in the board UI and documents when to use monitors versus `blocked`. > - The benefit is clearer liveness semantics for asynchronous waits without weakening single-assignee task ownership. ## What Changed - Added issue monitor fields, shared types, validators, constants, and an idempotent `0075` migration for scheduled monitor state. - Added server-side monitor scheduling, dispatch, recovery bounds, activity logging, and external-ref redaction. - Added board/agent route coverage for monitor permissions and child monitor scheduling. - Added issue detail/property UI for monitor state, a monitor activity card, and Storybook stories for review surfaces. - Documented monitor semantics and recovery policy behavior in `doc/execution-semantics.md`. - Addressed Greptile review feedback by preserving monitor state in skipped-stage builders and making board monitor saves send `scheduledBy: "board"`. ## Verification - `pnpm install --frozen-lockfile` - `pnpm run preflight:workspace-links && pnpm exec vitest run server/src/__tests__/issue-execution-policy-routes.test.ts server/src/__tests__/issue-execution-policy.test.ts server/src/__tests__/issue-monitor-scheduler.test.ts server/src/__tests__/recovery-classifiers.test.ts ui/src/components/IssueMonitorActivityCard.test.tsx ui/src/components/IssueProperties.test.tsx ui/src/lib/activity-format.test.ts` - First run passed 5 files and failed to collect 2 server suites because the worktree was missing the optional `acpx/runtime` dependency. - After `pnpm install --frozen-lockfile`, reran the 2 failed suites successfully. - `pnpm exec vitest run server/src/__tests__/issue-monitor-scheduler.test.ts server/src/__tests__/recovery-classifiers.test.ts` - `pnpm --filter @paperclipai/shared typecheck && pnpm --filter @paperclipai/db typecheck && pnpm --filter @paperclipai/server typecheck && pnpm --filter @paperclipai/ui typecheck` - `pnpm exec vitest run server/src/__tests__/issue-execution-policy.test.ts ui/src/components/IssueProperties.test.tsx` - `pnpm --filter @paperclipai/server typecheck && pnpm --filter @paperclipai/ui typecheck` - `pnpm exec vitest run ui/src/components/IssueMonitorActivityCard.test.tsx ui/src/components/IssueProperties.test.tsx` - `pnpm --filter @paperclipai/ui typecheck` - Storybook screenshot captured from `http://127.0.0.1:6006/iframe.html?viewMode=story&id=product-issue-monitor-surfaces--monitor-surfaces` with Playwright. ## Screenshots  ## Risks - Medium: this changes heartbeat recovery behavior for scheduled external-service waits, so regressions could affect wake timing or recovery issue creation. - Migration risk is reduced by using `IF NOT EXISTS` for the new issue monitor columns and index. - External monitor references are treated as secret-adjacent and are intentionally omitted from visible activity/wake payloads. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5 coding agent with repository tool use and terminal execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] If this change affects the UI, I have included before/after screenshots or Storybook review surfaces - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
GitHub
parent
76f09c8eb6
commit
57229d0f24
32 changed files with 19324 additions and 20 deletions
|
|
@ -7,6 +7,7 @@ const mockIssueService = vi.hoisted(() => ({
|
|||
getById: vi.fn(),
|
||||
assertCheckoutOwner: vi.fn(),
|
||||
update: vi.fn(),
|
||||
createChild: vi.fn(),
|
||||
addComment: vi.fn(),
|
||||
findMentionedAgents: vi.fn(),
|
||||
getRelationSummaries: vi.fn(),
|
||||
|
|
@ -16,21 +17,26 @@ const mockIssueService = vi.hoisted(() => ({
|
|||
|
||||
const mockHeartbeatService = vi.hoisted(() => ({
|
||||
wakeup: vi.fn(async () => undefined),
|
||||
triggerIssueMonitor: vi.fn(async () => ({ outcome: "triggered" as const })),
|
||||
reportRunActivity: vi.fn(async () => undefined),
|
||||
getRun: vi.fn(async () => null),
|
||||
getActiveRunForAgent: vi.fn(async () => null),
|
||||
cancelRun: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
const mockAccessService = vi.hoisted(() => ({
|
||||
canUser: vi.fn(async () => false),
|
||||
hasPermission: vi.fn(async () => false),
|
||||
}));
|
||||
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
companyService: () => ({
|
||||
getById: vi.fn(async () => ({ id: "company-1", attachmentMaxBytes: 10 * 1024 * 1024 })),
|
||||
}),
|
||||
accessService: () => ({
|
||||
canUser: vi.fn(async () => false),
|
||||
hasPermission: vi.fn(async () => false),
|
||||
}),
|
||||
accessService: () => mockAccessService,
|
||||
agentService: () => ({
|
||||
getById: vi.fn(async () => null),
|
||||
}),
|
||||
|
|
@ -42,6 +48,9 @@ function registerModuleMocks() {
|
|||
}),
|
||||
goalService: () => ({}),
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
environmentService: () => ({
|
||||
getById: vi.fn(async () => null),
|
||||
}),
|
||||
instanceSettingsService: () => ({
|
||||
get: vi.fn(async () => ({
|
||||
id: "instance-settings-1",
|
||||
|
|
@ -67,7 +76,7 @@ function registerModuleMocks() {
|
|||
syncIssue: async () => undefined,
|
||||
}),
|
||||
issueService: () => mockIssueService,
|
||||
logActivity: vi.fn(async () => undefined),
|
||||
logActivity: mockLogActivity,
|
||||
projectService: () => ({}),
|
||||
routineService: () => ({
|
||||
syncRunStatusForIssue: vi.fn(async () => undefined),
|
||||
|
|
@ -76,7 +85,22 @@ function registerModuleMocks() {
|
|||
}));
|
||||
}
|
||||
|
||||
async function createApp() {
|
||||
type TestActor =
|
||||
| {
|
||||
type: "board";
|
||||
userId: string;
|
||||
companyIds: string[];
|
||||
source: "local_implicit";
|
||||
isInstanceAdmin: boolean;
|
||||
}
|
||||
| {
|
||||
type: "agent";
|
||||
agentId: string;
|
||||
companyId: string;
|
||||
runId: string | null;
|
||||
};
|
||||
|
||||
async function createApp(actor?: TestActor) {
|
||||
const [{ errorHandler }, { issueRoutes }] = await Promise.all([
|
||||
import("../middleware/index.js"),
|
||||
import("../routes/issues.js"),
|
||||
|
|
@ -84,7 +108,7 @@ async function createApp() {
|
|||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = {
|
||||
(req as any).actor = actor ?? {
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
companyIds: ["company-1"],
|
||||
|
|
@ -111,6 +135,17 @@ describe("issue execution policy routes", () => {
|
|||
mockIssueService.getRelationSummaries.mockResolvedValue({ blockedBy: [], blocks: [] });
|
||||
mockIssueService.listWakeableBlockedDependents.mockResolvedValue([]);
|
||||
mockIssueService.getWakeableParentAfterChildCompletion.mockResolvedValue(null);
|
||||
mockIssueService.createChild.mockResolvedValue({
|
||||
issue: {
|
||||
id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
|
||||
companyId: "company-1",
|
||||
identifier: "PAP-1002",
|
||||
title: "Child issue",
|
||||
},
|
||||
parentBlockerAdded: false,
|
||||
});
|
||||
mockAccessService.canUser.mockResolvedValue(false);
|
||||
mockAccessService.hasPermission.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it("does not auto-start execution review when reviewers are added to an already in_review issue", async () => {
|
||||
|
|
@ -162,4 +197,175 @@ describe("issue execution policy routes", () => {
|
|||
expect(updatePatch.executionState).toBeUndefined();
|
||||
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("triggers a scheduled monitor immediately from the dedicated route", async () => {
|
||||
const issue = {
|
||||
id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
companyId: "company-1",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
assigneeUserId: null,
|
||||
createdByUserId: "local-board",
|
||||
identifier: "PAP-1001",
|
||||
title: "Manual monitor trigger",
|
||||
executionPolicy: normalizeIssueExecutionPolicy({
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
scheduledBy: "board",
|
||||
},
|
||||
}),
|
||||
executionState: null,
|
||||
};
|
||||
mockIssueService.getById.mockResolvedValue(issue);
|
||||
|
||||
const res = await request(await createApp())
|
||||
.post("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/monitor/check-now")
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ ok: true });
|
||||
expect(mockHeartbeatService.triggerIssueMonitor).toHaveBeenCalledWith(
|
||||
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
expect.objectContaining({
|
||||
actorType: "user",
|
||||
actorId: "local-board",
|
||||
agentId: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("lets a board user create a child issue with a scheduled monitor", async () => {
|
||||
mockIssueService.getById.mockResolvedValue({
|
||||
id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
companyId: "company-1",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: "11111111-1111-4111-8111-111111111111",
|
||||
assigneeUserId: null,
|
||||
createdByUserId: "local-board",
|
||||
identifier: "PAP-1001",
|
||||
title: "Parent issue",
|
||||
executionPolicy: null,
|
||||
executionState: null,
|
||||
});
|
||||
|
||||
const res = await request(await createApp())
|
||||
.post("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/children")
|
||||
.send({
|
||||
title: "Child monitor",
|
||||
status: "in_review",
|
||||
assigneeAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
executionPolicy: {
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
scheduledBy: "assignee",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
const createPayload = mockIssueService.createChild.mock.calls[0]?.[1] as {
|
||||
executionPolicy: { monitor: { scheduledBy: string } };
|
||||
};
|
||||
expect(createPayload.executionPolicy.monitor.scheduledBy).toBe("board");
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.monitor_scheduled",
|
||||
details: expect.objectContaining({
|
||||
scheduledBy: "board",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects child monitor scheduling by a non-assignee agent even with task assignment permission", async () => {
|
||||
mockAccessService.hasPermission.mockResolvedValue(true);
|
||||
mockIssueService.getById.mockResolvedValue({
|
||||
id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
companyId: "company-1",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: "11111111-1111-4111-8111-111111111111",
|
||||
assigneeUserId: null,
|
||||
createdByUserId: "local-board",
|
||||
identifier: "PAP-1001",
|
||||
title: "Parent issue",
|
||||
executionPolicy: null,
|
||||
executionState: null,
|
||||
});
|
||||
|
||||
const res = await request(await createApp({
|
||||
type: "agent",
|
||||
agentId: "22222222-2222-4222-8222-222222222222",
|
||||
companyId: "company-1",
|
||||
runId: "run-1",
|
||||
}))
|
||||
.post("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/children")
|
||||
.send({
|
||||
title: "Child monitor",
|
||||
status: "in_review",
|
||||
assigneeAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
executionPolicy: {
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
scheduledBy: "board",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.error).toBe("Only the assignee agent or a board user can manage issue monitors");
|
||||
expect(mockIssueService.createChild).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("normalizes spoofed child monitor scheduledBy to the assignee actor", async () => {
|
||||
mockAccessService.hasPermission.mockResolvedValue(true);
|
||||
mockIssueService.getById.mockResolvedValue({
|
||||
id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
|
||||
companyId: "company-1",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
assigneeUserId: null,
|
||||
createdByUserId: "local-board",
|
||||
identifier: "PAP-1001",
|
||||
title: "Parent issue",
|
||||
executionPolicy: null,
|
||||
executionState: null,
|
||||
});
|
||||
|
||||
const res = await request(await createApp({
|
||||
type: "agent",
|
||||
agentId: "33333333-3333-4333-8333-333333333333",
|
||||
companyId: "company-1",
|
||||
runId: "run-1",
|
||||
}))
|
||||
.post("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/children")
|
||||
.send({
|
||||
title: "Child monitor",
|
||||
status: "in_review",
|
||||
assigneeAgentId: "33333333-3333-4333-8333-333333333333",
|
||||
executionPolicy: {
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
scheduledBy: "board",
|
||||
externalRef: "https://example.test/deploy?token=secret",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
const createPayload = mockIssueService.createChild.mock.calls[0]?.[1] as {
|
||||
executionPolicy: { monitor: { scheduledBy: string; externalRef: string | null } };
|
||||
};
|
||||
expect(createPayload.executionPolicy.monitor.scheduledBy).toBe("assignee");
|
||||
expect(createPayload.executionPolicy.monitor.externalRef).toBe("[redacted]");
|
||||
expect(mockLogActivity).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
action: "issue.monitor_scheduled",
|
||||
entityId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
|
||||
details: expect.not.objectContaining({ externalRef: expect.anything() }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -112,6 +112,26 @@ describe("normalizeIssueExecutionPolicy", () => {
|
|||
it("throws for invalid input", () => {
|
||||
expect(() => normalizeIssueExecutionPolicy({ stages: [{ type: "invalid_type" }] })).toThrow();
|
||||
});
|
||||
|
||||
it("keeps monitor-only policies", () => {
|
||||
const result = normalizeIssueExecutionPolicy({
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
externalRef: "https://example.test/deploy?token=secret",
|
||||
},
|
||||
stages: [],
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
stages: [],
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
scheduledBy: "assignee",
|
||||
externalRef: "[redacted]",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseIssueExecutionState", () => {
|
||||
|
|
@ -1261,4 +1281,169 @@ describe("issue execution policy transitions", () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("monitor policy", () => {
|
||||
it("schedules a one-shot monitor on an active agent-owned issue", () => {
|
||||
const policy = normalizeIssueExecutionPolicy({
|
||||
stages: [],
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
scheduledBy: "board",
|
||||
},
|
||||
})!;
|
||||
|
||||
const result = applyIssueExecutionPolicyTransition({
|
||||
issue: {
|
||||
status: "in_progress",
|
||||
assigneeAgentId: coderAgentId,
|
||||
assigneeUserId: null,
|
||||
executionPolicy: null,
|
||||
executionState: null,
|
||||
monitorAttemptCount: 0,
|
||||
monitorNextCheckAt: null,
|
||||
monitorLastTriggeredAt: null,
|
||||
monitorNotes: null,
|
||||
monitorScheduledBy: null,
|
||||
},
|
||||
policy,
|
||||
previousPolicy: null,
|
||||
requestedAssigneePatch: {},
|
||||
actor: { userId: boardUserId },
|
||||
monitorExplicitlyUpdated: true,
|
||||
});
|
||||
|
||||
expect(result.patch.monitorNextCheckAt).toEqual(new Date("2026-04-11T12:30:00.000Z"));
|
||||
expect(result.patch.monitorScheduledBy).toBe("board");
|
||||
expect(result.patch.executionState).toMatchObject({
|
||||
status: "idle",
|
||||
monitor: {
|
||||
status: "scheduled",
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
scheduledBy: "board",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("auto-clears a scheduled monitor when the issue moves to done", () => {
|
||||
const policy = normalizeIssueExecutionPolicy({
|
||||
stages: [],
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
scheduledBy: "assignee",
|
||||
},
|
||||
})!;
|
||||
|
||||
const result = applyIssueExecutionPolicyTransition({
|
||||
issue: {
|
||||
status: "in_progress",
|
||||
assigneeAgentId: coderAgentId,
|
||||
assigneeUserId: null,
|
||||
executionPolicy: policy,
|
||||
executionState: {
|
||||
status: "idle",
|
||||
currentStageId: null,
|
||||
currentStageIndex: null,
|
||||
currentStageType: null,
|
||||
currentParticipant: null,
|
||||
returnAssignee: null,
|
||||
completedStageIds: [],
|
||||
lastDecisionId: null,
|
||||
lastDecisionOutcome: null,
|
||||
monitor: {
|
||||
status: "scheduled",
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
lastTriggeredAt: null,
|
||||
attemptCount: 0,
|
||||
notes: "Check deployment",
|
||||
scheduledBy: "assignee",
|
||||
clearedAt: null,
|
||||
clearReason: null,
|
||||
},
|
||||
},
|
||||
monitorAttemptCount: 0,
|
||||
monitorNextCheckAt: new Date("2026-04-11T12:30:00.000Z"),
|
||||
monitorLastTriggeredAt: null,
|
||||
monitorNotes: "Check deployment",
|
||||
monitorScheduledBy: "assignee",
|
||||
},
|
||||
policy,
|
||||
previousPolicy: policy,
|
||||
requestedStatus: "done",
|
||||
requestedAssigneePatch: {},
|
||||
actor: { agentId: coderAgentId },
|
||||
});
|
||||
|
||||
expect(result.patch.executionPolicy).toBeNull();
|
||||
expect(result.patch.monitorNextCheckAt).toBeNull();
|
||||
expect(result.patch.executionState).toMatchObject({
|
||||
monitor: {
|
||||
status: "cleared",
|
||||
clearReason: "done",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects explicitly scheduling a monitor on an invalid issue state", () => {
|
||||
const policy = normalizeIssueExecutionPolicy({
|
||||
stages: [],
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-11T12:30:00.000Z",
|
||||
notes: "Check deployment",
|
||||
},
|
||||
})!;
|
||||
|
||||
expect(() =>
|
||||
applyIssueExecutionPolicyTransition({
|
||||
issue: {
|
||||
status: "blocked",
|
||||
assigneeAgentId: coderAgentId,
|
||||
assigneeUserId: null,
|
||||
executionPolicy: null,
|
||||
executionState: null,
|
||||
},
|
||||
policy,
|
||||
previousPolicy: null,
|
||||
requestedAssigneePatch: {},
|
||||
actor: { agentId: coderAgentId },
|
||||
monitorExplicitlyUpdated: true,
|
||||
}),
|
||||
).toThrow("Monitor can only be scheduled");
|
||||
});
|
||||
|
||||
it("rejects explicitly re-arming a monitor after max attempts are exhausted", () => {
|
||||
const policy = normalizeIssueExecutionPolicy({
|
||||
stages: [],
|
||||
monitor: {
|
||||
nextCheckAt: "2099-04-11T12:30:00.000Z",
|
||||
maxAttempts: 1,
|
||||
scheduledBy: "assignee",
|
||||
},
|
||||
})!;
|
||||
|
||||
expect(() =>
|
||||
applyIssueExecutionPolicyTransition({
|
||||
issue: {
|
||||
status: "in_review",
|
||||
assigneeAgentId: coderAgentId,
|
||||
assigneeUserId: null,
|
||||
executionPolicy: null,
|
||||
executionState: null,
|
||||
monitorAttemptCount: 1,
|
||||
monitorNextCheckAt: null,
|
||||
monitorLastTriggeredAt: null,
|
||||
monitorNotes: null,
|
||||
monitorScheduledBy: "assignee",
|
||||
},
|
||||
policy,
|
||||
previousPolicy: null,
|
||||
requestedAssigneePatch: {},
|
||||
actor: { agentId: coderAgentId },
|
||||
monitorExplicitlyUpdated: true,
|
||||
}),
|
||||
).toThrow("Monitor bounds are already exhausted");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
448
server/src/__tests__/issue-monitor-scheduler.test.ts
Normal file
448
server/src/__tests__/issue-monitor-scheduler.test.ts
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
agentRuntimeState,
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
companies,
|
||||
companySkills,
|
||||
createDb,
|
||||
documentRevisions,
|
||||
documents,
|
||||
environmentLeases,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
issues,
|
||||
workspaceRuntimeServices,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { normalizeIssueExecutionPolicy, parseIssueExecutionState } from "../services/issue-execution-policy.ts";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres issue monitor scheduler tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("issue monitor scheduler", () => {
|
||||
let db!: ReturnType<typeof createDb>;
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
const seededAgentIds = new Set<string>();
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-monitor-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
|
||||
async function waitForHeartbeatIdle(timeoutMs = 3_000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const active = await db
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(sql`${heartbeatRuns.status} in ('queued', 'running', 'scheduled_retry')`);
|
||||
if (active.length === 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
throw new Error("Timed out waiting for issue monitor heartbeat runs to settle");
|
||||
}
|
||||
|
||||
async function heartbeatSideEffectFingerprint() {
|
||||
const [active, events, activity, leases, runtimeServices] = await Promise.all([
|
||||
db
|
||||
.select({ count: sql<number>`count(*)` })
|
||||
.from(heartbeatRuns)
|
||||
.where(sql`${heartbeatRuns.status} in ('queued', 'running', 'scheduled_retry')`),
|
||||
db.select({ count: sql<number>`count(*)` }).from(heartbeatRunEvents),
|
||||
db.select({ count: sql<number>`count(*)` }).from(activityLog),
|
||||
db.select({ count: sql<number>`count(*)` }).from(environmentLeases),
|
||||
db.select({ count: sql<number>`count(*)` }).from(workspaceRuntimeServices),
|
||||
]);
|
||||
|
||||
return [
|
||||
active[0]?.count ?? 0,
|
||||
events[0]?.count ?? 0,
|
||||
activity[0]?.count ?? 0,
|
||||
leases[0]?.count ?? 0,
|
||||
runtimeServices[0]?.count ?? 0,
|
||||
].join(":");
|
||||
}
|
||||
|
||||
async function waitForHeartbeatSideEffectsSettled(timeoutMs = 5_000, quietMs = 500) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let previous = "";
|
||||
let stableSince = Date.now();
|
||||
while (Date.now() < deadline) {
|
||||
const current = await heartbeatSideEffectFingerprint();
|
||||
const activeCount = Number(current.split(":")[0] ?? 0);
|
||||
if (current !== previous || activeCount > 0) {
|
||||
previous = current;
|
||||
stableSince = Date.now();
|
||||
} else if (Date.now() - stableSince >= quietMs) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
throw new Error("Timed out waiting for issue monitor heartbeat side effects to settle");
|
||||
}
|
||||
|
||||
async function cleanupRows() {
|
||||
await waitForHeartbeatSideEffectsSettled();
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(documentRevisions);
|
||||
await db.delete(issueDocuments);
|
||||
await db.delete(documents);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(environmentLeases);
|
||||
await db.delete(workspaceRuntimeServices);
|
||||
await db.delete(issues);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(agentRuntimeState);
|
||||
await db.delete(agents);
|
||||
await db.delete(companySkills);
|
||||
await db.delete(companies);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
seededAgentIds.clear();
|
||||
let lastError: unknown = null;
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
try {
|
||||
await cleanupRows();
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedFixture(input?: {
|
||||
agentStatus?: "active" | "paused";
|
||||
issueStatus?: "in_progress" | "in_review";
|
||||
monitorAttemptCount?: number;
|
||||
monitor?: Record<string, unknown>;
|
||||
}) {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const issueId = randomUUID();
|
||||
const nextCheckAt = new Date("2026-04-11T12:30:00.000Z");
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
|
||||
const monitorAttemptCount = input?.monitorAttemptCount ?? 0;
|
||||
const monitor = {
|
||||
nextCheckAt: nextCheckAt.toISOString(),
|
||||
notes: "Check deploy",
|
||||
scheduledBy: "assignee",
|
||||
...(input?.monitor ?? {}),
|
||||
};
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Monitor Bot",
|
||||
role: "engineer",
|
||||
status: input?.agentStatus ?? "active",
|
||||
adapterType: "process",
|
||||
adapterConfig: {
|
||||
command: process.execPath,
|
||||
args: ["-e", ""],
|
||||
cwd: process.cwd(),
|
||||
},
|
||||
runtimeConfig: {
|
||||
heartbeat: {
|
||||
enabled: false,
|
||||
wakeOnDemand: true,
|
||||
},
|
||||
},
|
||||
permissions: {},
|
||||
});
|
||||
seededAgentIds.add(agentId);
|
||||
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Watch external deploy",
|
||||
status: input?.issueStatus ?? "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: agentId,
|
||||
issueNumber: 1,
|
||||
identifier: `${issuePrefix}-1`,
|
||||
executionPolicy: {
|
||||
mode: "normal",
|
||||
commentRequired: true,
|
||||
stages: [],
|
||||
monitor,
|
||||
},
|
||||
executionState: {
|
||||
status: "idle",
|
||||
currentStageId: null,
|
||||
currentStageIndex: null,
|
||||
currentStageType: null,
|
||||
currentParticipant: null,
|
||||
returnAssignee: null,
|
||||
completedStageIds: [],
|
||||
lastDecisionId: null,
|
||||
lastDecisionOutcome: null,
|
||||
monitor: {
|
||||
status: "scheduled",
|
||||
nextCheckAt: nextCheckAt.toISOString(),
|
||||
lastTriggeredAt: null,
|
||||
attemptCount: monitorAttemptCount,
|
||||
notes: "Check deploy",
|
||||
scheduledBy: "assignee",
|
||||
serviceName: typeof monitor.serviceName === "string" ? monitor.serviceName : null,
|
||||
externalRef: typeof monitor.externalRef === "string" ? monitor.externalRef : null,
|
||||
timeoutAt: typeof monitor.timeoutAt === "string" ? monitor.timeoutAt : null,
|
||||
maxAttempts: typeof monitor.maxAttempts === "number" ? monitor.maxAttempts : null,
|
||||
recoveryPolicy: typeof monitor.recoveryPolicy === "string" ? monitor.recoveryPolicy : null,
|
||||
clearedAt: null,
|
||||
clearReason: null,
|
||||
},
|
||||
},
|
||||
monitorNextCheckAt: nextCheckAt,
|
||||
monitorAttemptCount,
|
||||
monitorNotes: "Check deploy",
|
||||
monitorScheduledBy: "assignee",
|
||||
});
|
||||
|
||||
return { companyId, agentId, issueId, nextCheckAt };
|
||||
}
|
||||
|
||||
it("triggers due issue monitors once and clears the one-shot schedule", async () => {
|
||||
const { issueId, agentId } = await seedFixture();
|
||||
const heartbeat = heartbeatService(db);
|
||||
const tickAt = new Date("2026-04-11T12:31:00.000Z");
|
||||
|
||||
const result = await heartbeat.tickTimers(tickAt);
|
||||
|
||||
expect(result.enqueued).toBe(1);
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!);
|
||||
expect(issue.monitorNextCheckAt).toBeNull();
|
||||
expect(issue.monitorAttemptCount).toBe(1);
|
||||
expect(issue.monitorLastTriggeredAt?.toISOString()).toBe(tickAt.toISOString());
|
||||
expect(normalizeIssueExecutionPolicy(issue.executionPolicy ?? null)?.monitor ?? null).toBeNull();
|
||||
expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({
|
||||
status: "triggered",
|
||||
lastTriggeredAt: tickAt.toISOString(),
|
||||
attemptCount: 1,
|
||||
});
|
||||
|
||||
const wakeup = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(wakeup?.reason).toBe("issue_monitor_due");
|
||||
|
||||
const activity = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.entityId, issueId))
|
||||
.then((rows) => rows.map((row) => row.action));
|
||||
expect(activity).toContain("issue.monitor_triggered");
|
||||
});
|
||||
|
||||
it("lets the board trigger a scheduled issue monitor immediately", async () => {
|
||||
const { issueId, agentId, nextCheckAt } = await seedFixture();
|
||||
const heartbeat = heartbeatService(db);
|
||||
const triggeredAt = new Date("2026-04-11T12:00:00.000Z");
|
||||
|
||||
const result = await heartbeat.triggerIssueMonitor(issueId, {
|
||||
now: triggeredAt,
|
||||
actorType: "user",
|
||||
actorId: "local-board",
|
||||
});
|
||||
|
||||
expect(result.outcome).toBe("triggered");
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!);
|
||||
expect(issue.monitorNextCheckAt).toBeNull();
|
||||
expect(issue.monitorLastTriggeredAt?.toISOString()).toBe(triggeredAt.toISOString());
|
||||
expect(issue.monitorAttemptCount).toBe(1);
|
||||
expect(normalizeIssueExecutionPolicy(issue.executionPolicy ?? null)?.monitor ?? null).toBeNull();
|
||||
|
||||
const wakeup = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(wakeup?.reason).toBe("issue_monitor_due");
|
||||
expect(wakeup?.payload).toMatchObject({
|
||||
issueId,
|
||||
nextCheckAt: nextCheckAt.toISOString(),
|
||||
source: "manual",
|
||||
});
|
||||
|
||||
const activity = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.entityId, issueId))
|
||||
.orderBy(activityLog.createdAt);
|
||||
expect(activity.map((row) => row.action)).toContain("issue.monitor_triggered");
|
||||
const triggerEvent = activity.find((row) => row.action === "issue.monitor_triggered");
|
||||
expect(triggerEvent?.actorType).toBe("user");
|
||||
expect(triggerEvent?.actorId).toBe("local-board");
|
||||
expect(triggerEvent?.details).toMatchObject({
|
||||
nextCheckAt: nextCheckAt.toISOString(),
|
||||
source: "manual",
|
||||
});
|
||||
});
|
||||
|
||||
it("clears due monitors that cannot be dispatched and records a skip", async () => {
|
||||
const { issueId } = await seedFixture({ agentStatus: "paused" });
|
||||
const heartbeat = heartbeatService(db);
|
||||
const tickAt = new Date("2026-04-11T12:31:00.000Z");
|
||||
|
||||
const result = await heartbeat.tickTimers(tickAt);
|
||||
|
||||
expect(result.skipped).toBe(1);
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!);
|
||||
expect(issue.monitorNextCheckAt).toBeNull();
|
||||
expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({
|
||||
status: "cleared",
|
||||
clearReason: "dispatch_skipped",
|
||||
});
|
||||
|
||||
const activity = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.entityId, issueId))
|
||||
.then((rows) => rows.map((row) => row.action));
|
||||
expect(activity).toContain("issue.monitor_skipped");
|
||||
});
|
||||
|
||||
it("clears exhausted monitors and queues bounded owner recovery instead of another due check", async () => {
|
||||
const { issueId, agentId } = await seedFixture({
|
||||
monitorAttemptCount: 1,
|
||||
monitor: {
|
||||
maxAttempts: 1,
|
||||
recoveryPolicy: "wake_owner",
|
||||
},
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const tickAt = new Date("2026-04-11T12:31:00.000Z");
|
||||
|
||||
const result = await heartbeat.tickTimers(tickAt);
|
||||
|
||||
expect(result.enqueued).toBe(0);
|
||||
expect(result.skipped).toBe(1);
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!);
|
||||
expect(issue.monitorNextCheckAt).toBeNull();
|
||||
expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({
|
||||
status: "cleared",
|
||||
clearReason: "max_attempts_exhausted",
|
||||
});
|
||||
|
||||
const wakeup = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(wakeup?.reason).toBe("issue_monitor_recovery");
|
||||
expect(wakeup?.payload).toMatchObject({
|
||||
issueId,
|
||||
clearReason: "max_attempts_exhausted",
|
||||
maxAttempts: 1,
|
||||
});
|
||||
|
||||
const activity = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.entityId, issueId))
|
||||
.then((rows) => rows.map((row) => row.action));
|
||||
expect(activity).toContain("issue.monitor_exhausted");
|
||||
expect(activity).toContain("issue.monitor_recovery_wake_queued");
|
||||
expect(activity).not.toContain("issue.monitor_triggered");
|
||||
});
|
||||
|
||||
it("clears timed-out monitors and creates a visible recovery issue when requested", async () => {
|
||||
const { issueId, companyId } = await seedFixture({
|
||||
monitor: {
|
||||
timeoutAt: "2026-04-11T12:00:00.000Z",
|
||||
recoveryPolicy: "create_recovery_issue",
|
||||
},
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const tickAt = new Date("2026-04-11T12:31:00.000Z");
|
||||
|
||||
const result = await heartbeat.tickTimers(tickAt);
|
||||
|
||||
expect(result.enqueued).toBe(0);
|
||||
expect(result.skipped).toBe(1);
|
||||
|
||||
const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!);
|
||||
expect(issue.monitorNextCheckAt).toBeNull();
|
||||
expect(parseIssueExecutionState(issue.executionState)?.monitor).toMatchObject({
|
||||
status: "cleared",
|
||||
clearReason: "timeout_exceeded",
|
||||
});
|
||||
|
||||
const recoveryIssue = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(eq(issues.originId, issueId))
|
||||
.then((rows) => rows.find((row) => row.companyId === companyId && row.originKind === "stranded_issue_recovery") ?? null);
|
||||
expect(recoveryIssue).toMatchObject({
|
||||
parentId: issueId,
|
||||
priority: "high",
|
||||
});
|
||||
expect(["todo", "in_progress"]).toContain(recoveryIssue?.status);
|
||||
});
|
||||
|
||||
it("omits external monitor refs from wake payloads and activity details", async () => {
|
||||
const { issueId, agentId } = await seedFixture({
|
||||
monitor: {
|
||||
serviceName: "Deploy provider",
|
||||
externalRef: "https://provider.example/deploy/123?token=secret",
|
||||
},
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
const tickAt = new Date("2026-04-11T12:31:00.000Z");
|
||||
|
||||
await heartbeat.tickTimers(tickAt);
|
||||
|
||||
const wakeup = await db
|
||||
.select()
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.agentId, agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
expect(JSON.stringify(wakeup?.payload)).not.toContain("provider.example");
|
||||
expect(wakeup?.payload).not.toHaveProperty("externalRef");
|
||||
|
||||
const activity = await db
|
||||
.select()
|
||||
.from(activityLog)
|
||||
.where(eq(activityLog.entityId, issueId));
|
||||
expect(JSON.stringify(activity.map((row) => row.details))).not.toContain("provider.example");
|
||||
expect(activity.find((row) => row.action === "issue.monitor_triggered")?.details).not.toHaveProperty("externalRef");
|
||||
});
|
||||
});
|
||||
|
|
@ -74,6 +74,100 @@ describe("recovery classifier boundary", () => {
|
|||
expect(classifyIssueGraphLiveness(input)).toEqual(classifyIssueGraphLivenessCompat(input));
|
||||
});
|
||||
|
||||
it("treats a scheduled monitor as an explicit review action path", () => {
|
||||
const findings = classifyIssueGraphLiveness({
|
||||
now: "2026-04-30T18:00:00.000Z",
|
||||
issues: [
|
||||
{
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2945",
|
||||
title: "Wait for external review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
assigneeUserId: null,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: null,
|
||||
executionState: null,
|
||||
monitorNextCheckAt: "2026-04-30T19:00:00.000Z",
|
||||
},
|
||||
],
|
||||
relations: [],
|
||||
agents: [
|
||||
{
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
reportsTo: managerId,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(findings).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not treat overdue or exhausted monitors as explicit waiting paths", () => {
|
||||
const baseIssue = {
|
||||
id: issueId,
|
||||
companyId,
|
||||
identifier: "PAP-2945",
|
||||
title: "Wait for external review",
|
||||
status: "in_review",
|
||||
assigneeAgentId: agentId,
|
||||
assigneeUserId: null,
|
||||
createdByAgentId: null,
|
||||
createdByUserId: null,
|
||||
};
|
||||
const agents = [
|
||||
{
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
reportsTo: managerId,
|
||||
},
|
||||
];
|
||||
|
||||
const overdue = classifyIssueGraphLiveness({
|
||||
now: "2026-04-30T20:00:00.000Z",
|
||||
issues: [
|
||||
{
|
||||
...baseIssue,
|
||||
executionState: null,
|
||||
monitorNextCheckAt: "2026-04-30T19:00:00.000Z",
|
||||
},
|
||||
],
|
||||
relations: [],
|
||||
agents,
|
||||
});
|
||||
|
||||
const exhausted = classifyIssueGraphLiveness({
|
||||
now: "2026-04-30T18:00:00.000Z",
|
||||
issues: [
|
||||
{
|
||||
...baseIssue,
|
||||
executionPolicy: {
|
||||
monitor: {
|
||||
nextCheckAt: "2026-04-30T19:00:00.000Z",
|
||||
maxAttempts: 1,
|
||||
},
|
||||
},
|
||||
executionState: null,
|
||||
monitorNextCheckAt: "2026-04-30T19:00:00.000Z",
|
||||
monitorAttemptCount: 1,
|
||||
},
|
||||
],
|
||||
relations: [],
|
||||
agents,
|
||||
});
|
||||
|
||||
expect(overdue[0]?.state).toBe("in_review_without_action_path");
|
||||
expect(exhausted[0]?.state).toBe("in_review_without_action_path");
|
||||
});
|
||||
|
||||
it("keeps run liveness continuation decision parity with the compatibility export", () => {
|
||||
const input = {
|
||||
run: {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue