fix(ssh): Fix race condition in ssh and add better debug info on failures

Author: artemyarulin

tests/ssh/app-ssh.test.ts

@@ -94,6 +94,7 @@ const setupApp = async (env: TestEnv) => {
 async function sshShellExec(
   conn: Client,
   command: string,
+  allowNonZeroExitCode = false,
 ): Promise<{ code: number; stdout: string; stderr: string }> {
   const START = `__START_${Math.random().toString(36).slice(2)}__`;
   const END = `__END_${Math.random().toString(36).slice(2)}__`;
@@ -103,9 +104,13 @@ async function sshShellExec(
       if (err) return reject(err);
       let stdout = "";
       let stderr = "";
-      let code = -1;
+      let code = undefined;
       let done = false;
+
       const parseCode = (buf: string): boolean => {
+        if (code !== undefined) {
+          return true; // exit code was set already
+        }
         const idx = buf.indexOf(`${END}:`);
         if (idx === -1) return false;
         const tail = buf.substring(idx + END.length + 1).trim();
@@ -143,26 +148,41 @@ async function sshShellExec(
         } else if (startIdx !== -1) {
           cmdOut = stdout.substring(startIdx + START.length);
         }
+        if (code === undefined) {
+          return reject(
+            new Error(
+              `ssh error: command=${command}, stream closed before exit code marker was observed, stdout=${cmdOut}, stderr=${stderr}`,
+            ),
+          );
+        }
+        if (!allowNonZeroExitCode && code !== 0) {
+          return reject(
+            new Error(
+              `ssh error: command=${command}, code=${code}, stdout=${cmdOut}, stderr=${stderr}`,
+            ),
+          );
+        }
         stream.end();
         resolve({ code, stdout: cmdOut, stderr });
       };
       const onStdout = (d: Buffer) => {
         stdout += d.toString();
         if (parseCode(stdout)) {
-          finish();
+          stream.end();
         }
       };
       const onStderr = (d: Buffer) => {
         const t = d.toString();
         stderr += t;
         if (parseCode(stderr)) {
-          finish();
+          stream.end();
         }
       };
       stream.on("data", onStdout);
       stream.stderr.on("data", onStderr);
       stream.on("close", () => {
         console.log("stream close");
+        finish();
       });
       stream.write(`echo ${START}\n`);
       stream.write(`${command}\n`);
@@ -256,7 +276,7 @@ test("app-ssh", async () => {
     expect(checkData.code).toBe(0);
     expect(checkData.stdout).toContain("ok");
 
-    const errCase = await sshShellExec(conn, "echo oops 1>&2; false");
+    const errCase = await sshShellExec(conn, "echo oops 1>&2; false", true);
     expect(errCase.code).not.toBe(0);
     expect(errCase.stderr).toContain("oops");
   } finally {

tests/ssh/ssh.test.ts

@@ -7,7 +7,6 @@ test("ssh", async () => {
     const output = await env.runWasmerCommand({
       args: ["ssh", ...args],
       stdin,
-      noAssertSuccess: true,
     });
     const stdout = output.stdout.replace("\r\n", "\n").trim();
     return stdout;

tests/superpanics/python.test.ts

@@ -11,7 +11,12 @@ test.concurrent("deploy python app", async () => {
     i++;
     try {
       const env = TestEnv.fromEnv();
-      const filePath = path.join(projectRoot, "fixtures", "python", "echo-server.py");
+      const filePath = path.join(
+        projectRoot,
+        "fixtures",
+        "python",
+        "echo-server.py",
+      );
       let testCode = await fs.promises.readFile(filePath, "utf-8");
       testCode = testCode.replaceAll("__TEMPLATE__", `${Math.random()}`);
       console.log(testCode);

tests/validation/log.test.ts

@@ -12,7 +12,12 @@ import { projectRoot } from "../utils/path";
 
 describe("Log tests", () => {
   it("Check fetch is logged on simple logging app", async () => {
-    const filePath = path.join(projectRoot, "fixtures", "php", "path-logger.php");
+    const filePath = path.join(
+      projectRoot,
+      "fixtures",
+      "php",
+      "path-logger.php",
+    );
     const phpPathLogger = await fs.readFile(filePath, "utf-8");
     const env = TestEnv.fromEnv();
     const appName = randomAppName();