blobsets/src/blobsets/filestores.nim

import ../blobsets
import tiger
import std/asyncfile, std/asyncdispatch, std/os

proc ingestFile*(store: BlobStore; path: string): Future[tuple[id: BlobId, size: BiggestInt]] {.async.} =
  ## Ingest a file and return blob metadata.
  let
    file = openAsync(path, fmRead)
    fileSize = file.getFileSize
  defer:
    close file
  let stream = store.openIngestStream(fileSize, dataBlob)
  if fileSize > 0:
    var buf = newString(min(8 shl 10, fileSize))
    while true:
      let n = await file.readBuffer(buf[0].addr, buf.len)
      if n == 0: break
      await stream.ingest(buf[0].addr, n)
  return await finish stream

type
  FsBlobStream = ref FsBlobStreamObj
  FsBlobStreamObj = object of BlobStreamObj
    path: string
    file: AsyncFile

  FsIngestStream = ref FsIngestStreamObj
  FsIngestStreamObj = object of IngestStreamObj
    ctx: TigerState
    leaves: seq[BlobId]
    path: string
    file: AsyncFile
    pos, nodeOffset: BiggestInt

  FileStore* = ref FileStoreObj
    ## A store that writes nodes and leafs as files.
  FileStoreObj = object of BlobStoreObj
    root, buf: string

proc fsBlobClose(s: BlobStream) =
  var s = FsBlobStream(s)
  close s.file

proc fsBlobSize(s: BlobStream): BiggestInt =
  var s = FsBlobStream(s)
  s.file.getFileSize.BiggestInt

proc setPosFs(s: BlobStream; pos: BiggestInt) =
  var s = FsBlobStream(s)
  s.file.setFilePos (int64)pos

proc getPosFs(s: BlobStream): BiggestInt =
  var s = FsBlobStream(s)
  (BiggestInt)s.file.getFilePos

proc fsBlobRead(s: BlobStream; buffer: pointer; len: Natural): Future[int] =
  var s = FsBlobStream(s)
  s.file.readBuffer(buffer, len)

proc fsOpenBlobStream(s: BlobStore; id: BlobId; size: BiggestInt; kind: BlobKind): BlobStream =
  var fs = FileStore(s)
  try:
    let
      path = fs.root / $kind / id.toBase32
      file = openAsync(path, fmRead)
    result = FsBlobStream(
      closeImpl: fsBlobClose,
      sizeImpl: fsBlobSize,
      setPosImpl: setPosFs,
      getPosImpl: getPosFs,
      readImpl: fsBlobRead,
      path: path, file: file,
    )
  except:
    raise newException(KeyError, "blob not in file-system store")

proc fsFinish(s: IngestStream): Future[tuple[id: BlobId, size: BiggestInt]] =
  var
    s = FsIngestStream(s)
    pair: tuple[id: BlobId, size: BiggestInt]
  close s.file
  if s.pos == 0 or s.pos mod blobLeafSize != 0:
    s.leaves.add finish(s.ctx)
  compressTree(s.leaves)
  pair.id = s.leaves[0]
  pair.size = s.pos
  let finalPath = s.path.parentDir / pair.id.toBase32
  if fileExists finalPath:
    removeFile s.path
  else:
    moveFile(s.path, finalPath)
  result = newFuture[tuple[id: BlobId, size: BiggestInt]]()
  complete result, pair

proc appendLeaf(s: FsIngestStream) =
  s.leaves.add(finish s.ctx)
  init s.ctx
  s.ctx.update [0'u8]

proc fsIngest(s: IngestStream; data: pointer; size: Natural): Future[void] =
  let
    s = FsIngestStream(s)
    buf = cast[ptr UncheckedArray[byte]](data)
  result = s.file.writeBuffer(data, size)
  var dataOff: int
  let leafOff = s.pos.int mod blobLeafSize
  if leafOff != 0:
    let leafFill = min(blobLeafSize - leafOff, size)
    s.ctx.update(buf[0].addr, leafFill)
    dataOff.inc leafFill
    if leafFill < size:
      appendLeaf s
  while dataOff+blobLeafSize <= size:
    s.ctx.update(buf[dataOff].addr, blobLeafSize)
    dataOff.inc blobLeafSize
    appendLeaf s
  if dataOff != size:
    s.ctx.update(buf[dataOff].addr, size - dataOff)
  s.pos.inc size

proc fsOpenIngestStream(s: BlobStore; size: BiggestInt; kind: BlobKind): IngestStream =
  var fs = FileStore(s)
  let stream = FsIngestStream(
    finishImpl: fsFinish,
    ingestImpl: fsIngest,
    path: fs.root / $kind / "ingest"
  )
  try: stream.file = openAsync(stream.path, fmWrite)
  except: raise newException(OSError,
    "failed to create ingest stream at '" & stream.path & "' " & getCurrentExceptionMsg())
  if size > 0:
    stream.file.setFileSize(size)
    stream.leaves = newSeqOfCap[BlobId](leafCount size)
  else:
    stream.leaves = newSeq[BlobId]()
  init stream.ctx
  stream.ctx.update [0'u8]
  stream

proc fsContains(s: BlobStore; id: BlobId; kind: BlobKind): Future[bool] =
  var fs = FileStore(s)
  result = newFuture[bool]("blobsets.filestores.fsContains")
  let path = fs.root / $kind / id.toBase32
  try:
    close(openAsync(path, fmRead))
    result.complete(true)
  except:
    result.complete(false)

proc newFileStore*(root: string): BlobStore =
  ## Create a new store object backed by a file-system.
  try:
    createDir(root / $dataBlob)
    createDir(root / $metaBlob)
  except: discard
  FileStore(
    containsImpl: fsContains,
    openBlobStreamImpl: fsOpenBlobStream,
    openIngestStreamImpl: fsOpenIngestStream,
    root: root,
    buf: "",
  )