Get lazy reading working

diff --git a/.gitignore b/.gitignore
index e5566e3..e5061a5 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
+/a.json
 /a.logjson

diff --git a/Mutex.mjs b/Mutex.mjs
new file mode 100644 (file)
index 0000000..593245f
--- /dev/null
+++ b/Mutex.mjs
@@ -0,0 +1,26 @@
+import assert from 'assert'
+
+class Mutex {
+  constructor() {
+    this.done = null
+  }
+
+  async acquire() {
+    while (this.done !== null)
+      await this.done.promise
+    let done = {}
+    done.promise = new Promise(
+      (resolve, reject) => {done.resolve = resolve}
+    )
+    this.done = done
+  }
+
+  release() {
+    let done = this.done
+    assert(done !== null)
+    this.done = null
+    done.resolve()
+  }
+}
+
+export default Mutex

diff --git a/a.mjs b/a.mjs
index d84e952..a6e4e51 100755 (executable)
--- a/a.mjs
+++ b/a.mjs
@@ -8,5 +8,4 @@ await file.open(
   'a.logjson', 
   JSON.parse(await fsPromises.readFile('random.json', 'utf-8'))
 )
-console.log('file.value', file.value)
 await file.close()

diff --git a/b.mjs b/b.mjs
index 7e89a37..4ef7c24 100755 (executable)
--- a/b.mjs
+++ b/b.mjs
@@ -5,5 +5,15 @@ import fsPromises from 'fs/promises'
 
 let file = new logjson.File()
 await file.open('a.logjson', {})
-console.log('file.value', file.value)
+await fsPromises.writeFile(
+  'a.json',
+  Buffer.from(
+    JSON.stringify(
+      await logjson.logjson_to_json(await file.get()),
+      null,
+      2
+    ) + '\n',
+    'utf-8'
+  )
+)
 await file.close()

diff --git a/logjson.mjs b/logjson.mjs
index 9c8c296..e2010c6 100644 (file)
--- a/logjson.mjs
+++ b/logjson.mjs
@@ -1,16 +1,21 @@
 import assert from 'assert'
 import fsPromises from 'fs/promises'
+import Mutex from './Mutex.mjs'
 
 let open_angle = Buffer.from('\n<', 'utf-8')
 let close_angle = Buffer.from('>\n', 'utf-8')
 class File {
   constructor() {
+    this.mutex = new Mutex()
     this.log = null
     this.eof = 0
-    this.value = null
+    this.ptr = 0
+    this.len = 0
+    this.value = undefined
   }
 
   async open(path, default_value) {
+    await this.mutex.acquire()
     assert(this.log === null)
 
     try {
@@ -23,24 +28,35 @@ class File {
       this.log = await fsPromises.open(path, 'w+')
       this.eof = 0
 
-      await this.write(default_value, true)
-      this.value = default_value
+      let {ptr, len} = await this.write(default_value, true)
+      this.ptr = ptr
+      this.len = len
+
+      // not yet, as it's JSON rather than logjson at the moment:
+      //this.value = default_value
       return
     }
 
     this.eof = (await this.log.stat()).size
     let {ptr, len} = await this.find_root()
-    this.value = await this.read(ptr, len)
+    this.ptr = ptr
+    this.len = len
 
-    this.eof = ptr + len + 2
+    // optional: trim any garbage off the end of the file
+    this.eof = this.ptr + this.len + 2
     await this.log.truncate(this.eof)
+
+    this.mutex.release()
   }
 
   async close() {
+    await this.mutex.acquire()
     assert(this.log !== null)
 
     await this.log.close()
     this.log = null
+
+    this.mutex.release()
   }
 
   async find_root() {
@@ -60,7 +76,7 @@ class File {
             return {ptr: start, len: end - start}
           }
 
-          if (ptr == 0)
+          if (ptr === 0)
             break
           ptr -= 0x1000
           if (count) {
@@ -83,7 +99,7 @@ class File {
         throw new Error('can\'t find logjson start marker')
       }
 
-      if (ptr == 0)
+      if (ptr === 0)
         break
       ptr -= 0x1000
       if (count) {
@@ -100,27 +116,6 @@ class File {
     throw new Error('can\'t find logjson end marker')
   }
 
-  async read(ptr, len) {
-    let buffer = Buffer.alloc(len)
-    assert((await this.log.read(buffer, 0, len, ptr)).bytesRead === len)
-    let value = JSON.parse(buffer.toString('utf-8'))
-    if (typeof value === 'object' && value !== null)
-      if (value instanceof Array) {
-        for (let i = 0; i < value.length; ++i) {
-          let item = value[i]
-          if (item instanceof Array)
-            value[i] = await this.read(item[0], item[1])
-        }
-      }
-      else
-        for (let i in value) {
-          let item = value[i]
-          if (item instanceof Array)
-            value[i] = await this.read(item[0], item[1])
-        }
-    return value
-  }
-
   async write(value, is_root) {
     if (typeof value === 'object' && value !== null)
       if (value instanceof Array) {
@@ -151,23 +146,221 @@ class File {
     let data = JSON.stringify(value, null, 2)
     if (is_root) {
       let buffer = Buffer.from(`<${data}>\n`)
+      let ptr = this.eof
       let len = buffer.length
       assert(
-        (await this.log.write(buffer, 0, len, this.eof)).bytesWritten === len
+        (await this.log.write(buffer, 0, len, ptr)).bytesWritten === len
       )
-      let ptr = this.eof + 1
       this.eof += len
-      return {ptr, len: len - 2}
+      return {ptr: ptr + 1, len: len - 3}
     }
     let buffer = Buffer.from(`${data}\n`)
+    let ptr = this.eof
     let len = buffer.length
     assert(
-      (await this.log.write(buffer, 0, len, this.eof)).bytesWritten === len
+      (await this.log.write(buffer, 0, len, ptr)).bytesWritten === len
     )
-    let ptr = this.eof
     this.eof += len
     return {ptr, len: len - 1}
   }
+
+  async get() {
+    if (this.value === undefined) {
+      let buffer = Buffer.alloc(this.len)
+      assert(
+        (await this.log.read(buffer, 0, this.len, this.ptr)).bytesRead ===
+          this.len
+      )
+      let value = JSON.parse(buffer.toString('utf-8'))
+      if (typeof value === 'object' && value !== null)
+        value =
+          value instanceof Array ?
+            new LazyArray(this, value) :
+            new LazyObject(this, value)
+      this.value = value
+    }
+    return this.value
+  }
+}
+
+// logjson array or object
+class Lazy {
+  constructor(file) {
+    this.file = file || null
+  }
+
+  has(key) {
+    throw new Error('not implemented')
+  }
+
+  async get(key, default_value) {
+    throw new Error('not implemented')
+  }
+
+  set(key, value) {
+    throw new Error('not implemented')
+  }
+
+  delete(key) {
+    throw new Error('not implemented')
+  }
+
+  keys() {
+    throw new Error('not implemented')
+  }
+}
+
+class LazyArray extends Lazy {
+  constructor(file, array) {
+    super(file)
+    this.array = array || []
+    this.length = this.array.length
+  }
+
+  has(key) {
+    assert(typeof key === 'number')
+    return key in this.array
+  }
+
+  async get(key, default_value) {
+    assert(typeof key === 'number')
+    if (!(key in this.array))
+      return default_value
+    let value = this.array[key]
+    if (value instanceof Array) {
+      let [ptr, len, new_value] = value
+      if (new_value === undefined) {
+        let buffer = Buffer.alloc(len)
+        assert(
+          (await this.file.log.read(buffer, 0, len, ptr)).bytesRead === len
+        )
+        new_value = JSON.parse(buffer.toString('utf-8'))
+        if (typeof new_value === 'object' && new_value !== null)
+          new_value =
+            new_value instanceof Array ?
+              new LazyArray(this.file, new_value) :
+              new LazyObject(this.file, new_value)
+        value[2] = new_value
+      }
+      value = new_value
+    }
+    return value
+  }
+
+  set(key, value) {
+    assert(typeof key === 'number')
+    if (typeof value === 'object' && value !== null) {
+      assert(value instanceof Lazy)
+      value = [-1, 0, value]
+    }
+    this.array[key] = value
+    this.length = this.array.length
+  }
+
+  delete(key) {
+    delete this.array[key]
+    this.length = this.array.length
+  }
+
+  keys() {
+    let keys = []
+    // this does not work well as it returns an array of strings:
+    //for (let i in this.array)
+    //  keys.push(i)
+    for (let i = 0; i < this.array.length; ++i)
+      if (i in this.array)
+        keys.push(i)
+    return keys
+  }
 }
 
-export default {File}
+class LazyObject extends Lazy {
+  constructor(file, object) {
+    super(file)
+    this.object = object || {}
+  }
+
+  has(key) {
+    assert(typeof key === 'string')
+    return Object.prototype.hasOwnProperty.call(this.object, key)
+  }
+
+  async get(key, default_value) {
+    assert(typeof key === 'string')
+    if (!Object.prototype.hasOwnProperty.call(this.object, key))
+      return default_value
+    let value = this.object[key]
+    if (value instanceof Array) {
+      let [ptr, len, new_value] = value
+      if (new_value === undefined) {
+        let buffer = Buffer.alloc(len)
+        assert(
+          (await this.file.log.read(buffer, 0, len, ptr)).bytesRead === len
+        )
+        new_value = JSON.parse(buffer.toString('utf-8'))
+        if (typeof new_value === 'object' && new_value !== null)
+          new_value =
+            new_value instanceof Array ?
+              new LazyArray(this.file, new_value) :
+              new LazyObject(this.file, new_value)
+        value[2] = new_value
+      }
+      value = new_value
+    }
+    return value
+  }
+
+  set(key, value) {
+    assert(typeof key === 'string')
+    if (typeof value === 'object' && value !== null) {
+      assert(value instanceof Lazy)
+      value = [-1, 0, value]
+    }
+    this.object[key] = value
+  }
+
+  delete(key) {
+    delete this.object[key]
+  }
+
+  keys() {
+    let keys = []
+    for (let i in this.object)
+      keys.push(i)
+    return keys
+  }
+}
+
+let logjson_to_json = async value => {
+  if (typeof value === 'object' && value !== null) {
+    assert(value instanceof Lazy)
+    let new_value = value instanceof LazyArray ? [] : {}
+    let keys = value.keys()
+    for (let i = 0; i < keys.length; ++i) {
+       let key = keys[i]
+       new_value[key] = await logjson_to_json(await value.get(key))
+    }
+    value = new_value
+  }
+  return value
+}
+
+let json_to_logjson = value => {
+  if (typeof value === 'object' && value !== null) {
+    let new_value =
+      value instanceof Array ? new LazyArray() : new LazyObject()
+    for (let i in value)
+      new_value.set(i, json_to_logjson(value[i]))
+    value = new_value
+  }
+  return value
+}
+
+export default {
+  File,
+  Lazy,
+  LazyArray,
+  LazyObject,
+  logjson_to_json,
+  json_to_logjson
+}