From 801d7c5ee02e1a79600ad5d6963fec31d451d489 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 27 Jan 2019 16:18:18 +0100 Subject: [PATCH] Improve error handling for HTTP server --- src/blobsets/filestores.nim | 52 ++++++++++-------- src/blobsets/httpservers.nim | 102 +++++++++++++++++++---------------- src/blobsets/httpstores.nim | 10 +++- 3 files changed, 92 insertions(+), 72 deletions(-) diff --git a/src/blobsets/filestores.nim b/src/blobsets/filestores.nim index f2a35ec..2beb70f 100644 --- a/src/blobsets/filestores.nim +++ b/src/blobsets/filestores.nim @@ -57,15 +57,19 @@ proc fsBlobRead(s: BlobStream; buffer: pointer; len: Natural): int = proc fsOpenBlobStream(s: BlobStore; id: BlobId; size: BiggestInt; kind: BlobKind): BlobStream = var fs = FileStore(s) - let path = fs.root / $kind / $id - FsBlobStream( - closeImpl: fsBlobClose, - setPosImpl: setPosFs, - getPosImpl: getPosFs, - readImpl: fsBlobRead, - path: path, - file: openAsync(path, fmRead), - ) + try: + let + path = fs.root / $kind / id.toHex + file = openAsync(path, fmRead) + result = FsBlobStream( + closeImpl: fsBlobClose, + setPosImpl: setPosFs, + getPosImpl: getPosFs, + readImpl: fsBlobRead, + path: path, file: file, + ) + except: + raise newException(KeyError, "blob not in file-system store") proc fsFinish(s: IngestStream): tuple[id: BlobId, size: BiggestInt] = var s = FsIngestStream(s) @@ -74,7 +78,7 @@ proc fsFinish(s: IngestStream): tuple[id: BlobId, size: BiggestInt] = compressTree(s.leaves) result.id = s.leaves[0] result.size = s.pos - moveFile(s.path, s.path.parentDir / $(result.id)) + moveFile(s.path, s.path.parentDir / result.id.toHex) proc fsIngest(s: IngestStream; buf: pointer; len: Natural) = var @@ -102,25 +106,27 @@ proc fsIngest(s: IngestStream; buf: pointer; len: Natural) = proc fsOpenIngestStream(s: BlobStore; size: BiggestInt; kind: BlobKind): IngestStream = var fs = FileStore(s) - let stream = FsIngestStream() - result = stream - stream.finishImpl = fsFinish - stream.ingestImpl = fsIngest - case kind - of dataBlob: - stream.path = fs.root / "data" / "ingest" - of metaBlob: - stream.path = fs.root / "blob" / "ingest" - stream.file = openAsync(stream.path, fmWrite) + let stream = FsIngestStream( + finishImpl: fsFinish, + ingestImpl: fsIngest, + path: fs.root / $kind / "ingest" + ) + try: stream.file = openAsync(stream.path, fmWrite) + except: raise newException(OSError, + "failed to create ingest stream at '" & stream.path & "' " & getCurrentExceptionMsg()) if size > 0: stream.file.setFileSize(size) stream.leaves = newSeqOfCap[BlobId](leafCount size) else: stream.leaves = newSeq[BlobId]() + stream -proc newFileStore*(root = "/"): FileStore = - createDir(root / "data") - createDir(root / "blob") +proc newFileStore*(root: string): FileStore = + ## Create a new store object backed by a file-system. + try: + createDir(root / $dataBlob) + createDir(root / $metaBlob) + except: discard new result result.openBlobStreamImpl = fsOpenBlobStream result.openIngestStreamImpl = fsOpenIngestStream diff --git a/src/blobsets/httpservers.nim b/src/blobsets/httpservers.nim index 93fd313..29324bb 100644 --- a/src/blobsets/httpservers.nim +++ b/src/blobsets/httpservers.nim @@ -13,6 +13,7 @@ type rng: Rand proc newHttpStoreServer*(backend: BlobStore): HttpStoreServer = + ## Create a new HTTP server for a given store. randomize() HttpStoreServer( server: newAsyncHttpServer(), @@ -31,45 +32,44 @@ func parseRange(range: string): tuple[a: int, b: int] = if result.b < result.a: reset result +proc blobStream(hss: HttpStoreServer; path: string): BlobStream = + let elems = path.split '/' + if not elems.len == 3: + raise newException(ValueError, "bad GET path " & path) + let + kind = case elems[1] + of $dataBlob: dataBlob + of $metaBlob: metaBlob + else: raise newException(ValueError, "bad GET path " & path) + blob = toBlobId elems[2] + if hss.blobs.contains blob: + result = hss.blobs[blob] + else: + result = hss.store.openBlobStream(blob, kind=kind) + hss.blobs[blob] = result + proc head(hss: HttpStoreServer; req: Request): Future[void] = - var stream: BlobStream - let elems = req.url.path.split '/' - if elems.len == 3: - let - kind = case elems[1] - of "data": dataBlob - of "meta": metaBlob - else: raise newException(ValueError, "bad GET path " & req.url.path) - blob = toBlobId elems[2] - var stream = hss.blobs.getOrDefault blob - if stream.isNil: - stream = hss.store.openBlobStream(blob, kind=kind) - hss.blobs[blob] = stream + discard hss.blobStream(req.url.path) + # cache the stream in the blob table or raise an exception req.respond(Http200, "") proc get(hss: HttpStoreServer; req: Request): Future[void] = - var stream: BlobStream - let elems = req.url.path.split '/' - if elems.len == 3: + let stream = hss.blobStream(req.url.path) + var + pos: BiggestInt + len = blobLeafSize + let range = req.headers.getOrDefault "range" + if range != "": let - kind = case elems[1] - of "data": dataBlob - of "meta": metaBlob - else: raise newException(ValueError, "bad GET path " & req.url.path) - blob = toBlobId elems[2] - var stream = hss.blobs.getOrDefault blob - if stream.isNil: - stream = hss.store.openBlobStream(blob, kind=kind) - hss.blobs[blob] = stream - let - (pos, endPos) = parseRange req.headers["range"] - len = endPos - pos - stream.pos = pos - var body = newString(len) - let n = stream.read(body[0].addr, len) - body.setLen n - return req.respond(Http206, body) - req.respond(Http404, "invalid request") + (startPos, endPos) = parseRange range + pos = startPos + len = endPos - startPos + stream.pos = pos + var body = newString(len) + len = stream.read(body[0].addr, len) + body.setLen len + let headers = newHttpHeaders({"Range": "bytes=$1-$2" % [ $pos, $(pos+len) ]}) + req.respond(Http206, body, headers) proc putIngest(hss: HttpStoreServer; req: Request): Future[void] = let stream = hss.ingests[req.url.path] @@ -78,14 +78,14 @@ proc putIngest(hss: HttpStoreServer; req: Request): Future[void] = proc postIngest(hss: HttpStoreServer; req: Request): Future[void] = case req.url.path - of "/ingest/data": + of "/ingest/" & $dataBlob: let size = parseBiggestInt req.headers["ingest-size"] key = "/ingest/" & $hss.rng.next headers = newHttpHeaders({"Location": key}) hss.ingests[key] = hss.store.openIngestStream(size, dataBlob) req.respond(Http201, "", headers) - of "/ingest/meta": + of "/ingest/" & $metaBlob: let size = parseBiggestInt req.headers["ingest-size"] key = "/ingest/" & $hss.rng.next @@ -104,16 +104,24 @@ proc postIngest(hss: HttpStoreServer; req: Request): Future[void] = req.respond(Http204, "", headers) proc serve*(hss: HttpStoreServer; port = Port(80)): Future[void] = + ## Serve requests to HTTP store. proc handleRequest(req: Request): Future[void] = - case req.reqMethod - of HttpGET: - get(hss, req) - of HttpHEAD: - head(hss, req) - of HttpPUT: - putIngest(hss, req) - of HttpPOST: - postIngest(hss, req) - else: - req.respond(Http501, "method not implemented") + try: + case req.reqMethod + of HttpGET: + get(hss, req) + of HttpHEAD: + head(hss, req) + of HttpPUT: + putIngest(hss, req) + of HttpPOST: + postIngest(hss, req) + else: + req.respond(Http501, "method not implemented") + except KeyError: + req.respond(Http404, "blob not found") + except ValueError: + req.respond(Http400, getCurrentExceptionMsg()) + except: + req.respond(Http500, getCurrentExceptionMsg()) hss.server.serve(port, handleRequest) diff --git a/src/blobsets/httpstores.nim b/src/blobsets/httpstores.nim index 581879f..b369b21 100644 --- a/src/blobsets/httpstores.nim +++ b/src/blobsets/httpstores.nim @@ -57,8 +57,14 @@ proc httpOpenBlobStream(store: BlobStore; id: BlobId; size: BiggestInt; kind: Bl url: $((store.url / $kind) / id.toHex) ) let resp = waitFor stream.client.request(stream.url, HttpHEAD) - assert(resp.code in {Http200, Http204}, resp.status) - stream + case resp.code + of Http200, Http204: + stream + of Http404: + raise newException(KeyError, "blob not at HTTP store") + else: + raise newException(KeyError, + "unrecognized " & $resp.code & " response from HTTP store: " & resp.status) proc httpFinish(s: IngestStream): tuple[id: BlobId, size: BiggestInt] = var s = HttpIngestStream(s)