From f2e38fe672ab413ba3cd05e868b2381408c36230 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 7 Feb 2025 17:26:32 -0500 Subject: [PATCH 1/2] Make server contemplate but not do GAF indexing, ban GAF uploads as unsupported, and allow upload errors to reach the user in an ugly way --- docker/config.json | 2 +- exampleData/cactus-NA12879-small.gaf | 100 +++++++++++++++++++++++++++ src/api/ServerAPI.mjs | 27 +++++--- src/components/HeaderForm.js | 6 +- src/components/TrackFilePicker.js | 27 +++++--- src/config.json | 2 +- src/server.mjs | 75 +++++++++++++++----- 7 files changed, 202 insertions(+), 37 deletions(-) create mode 100644 exampleData/cactus-NA12879-small.gaf diff --git a/docker/config.json b/docker/config.json index 764046fb..0bde79ad 100644 --- a/docker/config.json +++ b/docker/config.json @@ -84,7 +84,7 @@ "fileTypeToExtensions": { "graph": ".xg,.vg,.hg,.gbz,.pg,.db", "haplotype": ".gbwt,.gbz", - "read": ".gam,gaf.gz" + "read": ".gam" }, "MAXUPLOADSIZE": 5242880, diff --git a/exampleData/cactus-NA12879-small.gaf b/exampleData/cactus-NA12879-small.gaf new file mode 100644 index 00000000..29fb184a --- /dev/null +++ b/exampleData/cactus-NA12879-small.gaf @@ -0,0 +1,100 @@ +ERR194148.651218574/2 101 0 101 + >1>2 98 0 36 36 101 159 AS:i:36 bq:Z:BB1>2 98 0 72 72 101 254 AS:i:72 bq:Z:B<1>2 98 0 98 98 101 254 AS:i:98 bq:Z:BB1>2 98 0 82 82 101 254 AS:i:82 bq:Z:BB1>2 98 0 57 57 101 254 AS:i:57 bq:Z:BB1>2 98 0 30 30 101 124 AS:i:30 bq:Z:BB1>2 98 0 11 11 101 11 AS:i:11 bq:Z:B<1>2 98 0 72 72 101 254 AS:i:72 bq:Z:B<1>2 98 0 57 57 101 254 AS:i:57 bq:Z:BB1>2 98 0 58 58 101 254 AS:i:58 bq:Z:BB1>2 98 0 50 50 101 242 AS:i:50 bq:Z:BB1>2 98 0 21 21 101 254 AS:i:21 bq:Z:BB<1>2 98 0 21 21 101 254 AS:i:21 bq:Z:1>2 98 0 46 46 101 219 AS:i:46 bq:Z:BB1>2 98 0 50 50 101 242 AS:i:50 bq:Z:BB<<7BB7B1>2 98 0 62 62 101 254 AS:i:62 bq:Z:BB1>2 98 0 94 94 101 254 AS:i:94 bq:Z:BB1>2 98 0 72 72 101 254 AS:i:72 bq:Z:B<1>2 98 0 39 39 101 177 AS:i:39 bq:Z:BB<1>2 98 0 9 9 101 0 AS:i:9 bq:Z:BB<<1>2 98 0 92 92 101 254 AS:i:92 bq:Z:BB1>2 98 0 27 27 101 106 AS:i:27 bq:Z:BB1>2 98 0 60 60 101 254 AS:i:60 bq:Z:BB1>2 98 0 56 56 101 254 AS:i:56 bq:Z:BBBBBBBBBBBB7BBBBBBBBBBBBBBBBBBBB1>2 98 0 55 55 101 254 AS:i:55 bq:Z:BB1>2>3 194 0 100 100 101 254 AS:i:100 bq:Z:B<<1>2>3 194 0 101 101 101 254 AS:i:101 bq:Z:BB<2>3 192 55 156 101 101 254 AS:i:101 bq:Z:B7<2>3 192 64 165 101 101 254 AS:i:101 bq:Z:BB72>3 192 79 180 101 101 254 AS:i:101 bq:Z:B<<2>3 192 84 185 101 101 254 AS:i:101 bq:Z:BBBBBBBBBBBBBBBB<2>3 192 69 170 99 101 254 AS:i:91 bq:Z:B<<2>3 192 3 104 101 101 254 AS:i:101 bq:Z:2>3 192 64 165 101 101 254 AS:i:101 bq:Z:BB2>3 192 76 177 101 101 254 AS:i:101 bq:Z:BB2>3 192 60 161 101 101 254 AS:i:101 bq:Z:BB2>3 192 41 142 101 101 254 AS:i:101 bq:Z:BB<2>3 192 91 192 101 101 254 AS:i:101 bq:Z:BB2>3 192 35 136 101 101 254 AS:i:101 bq:Z:BB2>3 192 27 128 101 101 254 AS:i:101 bq:Z:BB<2>3 192 34 135 101 101 254 AS:i:101 bq:Z:B<2>3 192 58 159 101 101 254 AS:i:101 bq:Z:BB2>3 192 70 171 100 101 254 AS:i:96 bq:Z:BB2>3 192 28 129 101 101 254 AS:i:101 bq:Z:BB<2>3 192 70 171 99 101 254 AS:i:91 bq:Z:<7<0B0BBBB2>3 192 71 172 101 101 254 AS:i:101 bq:Z:BB2>3 192 64 165 101 101 254 AS:i:101 bq:Z:BB2>3 192 19 120 101 101 254 AS:i:101 bq:Z:BB<2>3 192 13 114 101 101 254 AS:i:101 bq:Z:BB2>3 192 46 147 101 101 254 AS:i:101 bq:Z:BB<<2>3 192 31 132 101 101 254 AS:i:101 bq:Z:B<<2>3 192 46 147 101 101 254 AS:i:101 bq:Z:BB<<2>3 192 16 117 101 101 254 AS:i:101 bq:Z:BB2>3 192 43 144 101 101 254 AS:i:101 bq:Z:BB<2>3 192 16 117 101 101 254 AS:i:101 bq:Z:BB2>3 192 18 119 101 101 254 AS:i:101 bq:Z:B<2>3 192 61 162 101 101 254 AS:i:101 bq:Z:BB<3 96 4 12 8 101 0 AS:i:8 bq:Z:BB3>4 192 23 124 101 101 254 AS:i:101 bq:Z:BB<3>4 192 85 186 101 101 254 AS:i:101 bq:Z:BB3>4 192 37 138 101 101 254 AS:i:101 bq:Z:<<<<7BBB3>4 192 13 114 101 101 254 AS:i:101 bq:Z:BB<3>4 192 91 192 101 101 254 AS:i:101 bq:Z:BB<3>4 192 79 177 97 101 118 AS:i:93 bq:Z:BBBBBBBBBBB<<3>4 192 79 177 97 101 0 AS:i:93 bq:Z:BBBBBBBBBBB<<3>4 192 56 157 101 101 254 AS:i:101 bq:Z:BB3>4 192 49 150 101 101 254 AS:i:101 bq:Z:BB { + return await new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); xhr.responseType = "json"; xhr.onreadystatechange = () => { @@ -121,21 +121,28 @@ export class ServerAPI extends APIInterface { reject(new Error("Upload aborted")); return; } - + if (xhr.readyState === 4) { if (xhr.status === 200 && xhr.response.path) { // Every thing ok, file uploaded, and we got a path. resolve(xhr.response.path); } else { // Something weird happened. - reject( - new Error( - "Failed to upload file: status " + - xhr.status + - " and response: " + - xhr.response - ) - ); + + if (xhr.response.error) { + // The server sent us a particular message. + reject(new Error(xhr.response.error)) + } else { + // The server did not help us. Compose a message. + reject( + new Error( + "Failed to upload file: status " + + xhr.status + + " and response: " + + JSON.stringify(xhr.response) + ) + ); + } } } }; diff --git a/src/components/HeaderForm.js b/src/components/HeaderForm.js index ad7f8798..6bb55e87 100644 --- a/src/components/HeaderForm.js +++ b/src/components/HeaderForm.js @@ -912,7 +912,8 @@ class HeaderForm extends Component { this.setState({ uploadInProgress: val }); }; - // Sends uploaded file to server and returns a path to the file + // Sends uploaded file to server and returns a path to the file, or raises an exception if the upload fails or is rejected. + // If the file upload is canceled, returns nothing. handleFileUpload = async (fileType, file) => { if (!(this.props.APIInterface instanceof LocalAPI) && file.size > config.MAXUPLOADSIZE) { this.showFileSizeAlert(); @@ -927,11 +928,14 @@ class HeaderForm extends Component { // Refresh the graphs right away this.getMountedFilenames(); } + // TODO: Is only one upload actually ever in progress at a time? Probably + // it's possible to have several!!! this.setUploadInProgress(false); return fileName; } catch (e) { if (!this.cancelSignal.aborted) { // Only pass along errors if we haven't canceled our fetches. + this.setUploadInProgress(false); throw e; } } diff --git a/src/components/TrackFilePicker.js b/src/components/TrackFilePicker.js index 82e6c84c..67b2ae5a 100644 --- a/src/components/TrackFilePicker.js +++ b/src/components/TrackFilePicker.js @@ -7,10 +7,9 @@ import { Input } from "reactstrap"; /* * A selection dropdown component that select files. - * Expects a file type object in the form of {"name": string, "type": string} + * Expects a collection of track objects in the form of {"name": string, "type": string} * - * The handleInputChange function expects to be passed an option type object in the form of - * {"label": string, "value": file} + * The handleInputChange function expects to be passed a bare value. * * See demo and test file for examples of this component. */ @@ -30,11 +29,23 @@ export const TrackFilePicker = ({ let acceptedExtensions = config.fileTypeToExtensions[fileType]; async function uploadOnChange() { - const file = uploadFileInput.current.files[0]; - - const completePath = await handleFileUpload(fileType, file); - console.log("TrackFilePicker got an upload result:", completePath); - handleInputChange(completePath); + // TODO: we have a ref here for a reason, but the ref's value can be null + // when the upload await finishes. So we capture it to a local. + const uploadingInput = uploadFileInput.current; + const file = uploadingInput.files[0]; + + try { + const completePath = await handleFileUpload(fileType, file); + console.log("TrackFilePicker got an upload result:", completePath); + // This will be nothing if the upoload was canceled. + handleInputChange(completePath); + } catch (e) { + console.error("TrackFilePicker could not upload: ", e); + // TODO: Display error to user in a better way + alert(e.toString()); + // Clear out the uploaded file to show it didn't work. + uploadingInput.value = null; + } } function mountedOnChange(option) { diff --git a/src/config.json b/src/config.json index 0ee6d782..f790df30 100644 --- a/src/config.json +++ b/src/config.json @@ -114,7 +114,7 @@ "fileTypeToExtensions": { "graph": ".xg,.vg,.hg,.gbz,.pg,.db", "haplotype": ".gbwt,.gbz", - "read": ".gam,.gaf.gz" + "read": ".gam" }, "MAXUPLOADSIZE": 5242880, diff --git a/src/server.mjs b/src/server.mjs index 4d8d8e57..98f60976 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -321,29 +321,66 @@ api.post("/trackFileSubmission", upload.single("trackFile"), (req, res) => { } }); -function indexGamSorted(req, res) { - const prefix = req.file.path.substring(0, req.file.path.lastIndexOf(".")); - const sortedGamFile = fs.createWriteStream(prefix + ".sorted.gam", { +function indexGamSorted(req, res, next) { + let readsPath = req.file.path; + let prefix; + let sortedSuffix; + let indexSuffix; + if (readsPath.endsWith(".gaf") || readsPath.endsWith(".gaf.gz")) { + throw new BadRequestError(`Server-side sorting and indexing not yet implemented for GAF: ${readsPath}`); + } else if (readsPath.endsWith(".gam")) { + prefix = readsPath.substring(0, req.file.path.lastIndexOf(".gam")); + sortedSuffix = ".sorted.gam"; + indexSuffix = ".sorted.gam.gai"; + } else { + throw new BadRequestError(`Read file is not a GAF or GAM: ${readsPath}`); + } + + const sortedReadsFile = fs.createWriteStream(prefix + sortedSuffix, { encoding: "binary", }); - const vgIndexChild = spawn(find_vg(), [ + + let vgGamsortParams = [ "gamsort", "-i", - prefix + ".sorted.gam.gai", + prefix + indexSuffix, req.file.path, - ]); + ]; + const vgGamsortChild = spawn(find_vg(), vgGamsortParams); + + req.error = Buffer.alloc(0); + + let sentResponse = false; + + vgGamsortChild.on("error", function (err) { + console.log( + "Error executing " + + find_vg() + " " + + vgGamsortParams.join(" ") + + ": " + + err + ); + if (!sentResponse) { + sentResponse = true; + return next(new VgExecutionError("vg gamsort failed")); + } + }); - vgIndexChild.stderr.on("data", (data) => { + vgGamsortChild.stderr.on("data", (data) => { console.log(`err data: ${data}`); + req.error += data; }); - vgIndexChild.stdout.on("data", function (data) { - sortedGamFile.write(data); + vgGamsortChild.stdout.on("data", function (data) { + sortedReadsFile.write(data); }); - vgIndexChild.on("close", () => { - sortedGamFile.end(); - res.json({ path: path.relative(".", prefix + ".sorted.gam") }); + vgGamsortChild.on("close", () => { + sortedReadsFile.end(); + if (!sentResponse) { + sentResponse = true; + res.json({ path: path.relative(".", prefix + sortedSuffix) }); + } }); } @@ -673,20 +710,25 @@ async function getChunkedData(req, res, next) { let anyGam = false; let anyGaf = false; for (const gamFile of gamFiles) { - if (!gamFile.endsWith(".gam") && !gamFile.endsWith(".gaf.gz")) { - throw new BadRequestError("GAM/GAF file doesn't end in .gam or .gaf.gz: " + gamFile); + if (!gamFile.endsWith(".gam") && !gamFile.endsWith(".gaf") && !gamFile.endsWith(".gaf.gz")) { + throw new BadRequestError("GAM/GAF file doesn't end in .gam, .gaf, or .gaf.gz: " + gamFile); } if (!isAllowedPath(gamFile)) { throw new BadRequestError("GAM/GAF file path not allowed: " + gamFile); } if (gamFile.endsWith(".gam")) { - // Use a GAM index + // Use a GAM console.log("pushing gam file", gamFile); anyGam = true; } + if (gamFile.endsWith(".gaf")) { + // Use a small GAF without an index + console.log("pushing gaf file", gamFile); + anyGaf = true; + } if (gamFile.endsWith(".gaf.gz")) { // Use a GAF with index - console.log("pushing gaf file", gamFile); + console.log("pushing hopefully indexed gaf file", gamFile); anyGaf = true; } vgChunkParams.push("-a", gamFile); @@ -1615,6 +1657,7 @@ api.get("/getFilenames", (req, res) => { if (file.endsWith(".sorted.gam")) { result.files.push({ trackFile: clientPath, trackType: "read" }); } + // We don't allow un-sorted-and-indexed plain GAF files here if (file.endsWith(".gaf.gz")) { result.files.push({"trackFile": file, "trackType": "read"}); } From 23a77c7f6dcf6c9dd302f21bca03da55885e23e8 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 7 Feb 2025 17:38:24 -0500 Subject: [PATCH 2/2] Hook up exit code to generate error and hook up error to Express --- src/server.mjs | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/server.mjs b/src/server.mjs index 98f60976..d7c2659c 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -311,14 +311,23 @@ api.use((req, res, next) => { }); // Store files uploaded from trackFilePicker via multer -api.post("/trackFileSubmission", upload.single("trackFile"), (req, res) => { - console.log("/trackFileSubmission"); - console.log(req.file); - if (req.body.fileType === fileTypes["READ"]) { - indexGamSorted(req, res); - } else { - res.json({ path: path.relative(".", req.file.path) }); - } +api.post("/trackFileSubmission", upload.single("trackFile"), (req, res, next) => { + // We would like this to be an async function, but then Express error + // handling doesn't work, because it doesn't detect returned promise + // rejections until Express 5. We have to pass an error to next() or else + // throw synchronously. + captureErrors(next, async () => { + console.log("/trackFileSubmission"); + console.log(req.file); + // We don't get a lock because we're putting new files in and so we don't + // need to block using them or cleaning old files. + + if (req.body.fileType === fileTypes["READ"]) { + indexGamSorted(req, res, next); + } else { + res.json({ path: path.relative(".", req.file.path) }); + } + }); }); function indexGamSorted(req, res, next) { @@ -375,8 +384,19 @@ function indexGamSorted(req, res, next) { sortedReadsFile.write(data); }); - vgGamsortChild.on("close", () => { + vgGamsortChild.on("close", (code) => { + console.log(`vg gamsort exited with code ${code}`); sortedReadsFile.end(); + + if (code !== 0) { + // Execution failed + if (!sentResponse) { + sentResponse = true; + return next(new VgExecutionError("vg gamsort failed")); + } + return; + } + if (!sentResponse) { sentResponse = true; res.json({ path: path.relative(".", prefix + sortedSuffix) });