let util = require('util')
let fs_readFile = util.promisify(fs.readFile)
+let fs_rename = util.promisify(fs.rename)
+let fs_unlink = util.promisify(fs.unlink)
let fs_writeFile = util.promisify(fs.writeFile)
let JSONCache = function(diag) {
this.diag = diag || false
}
-let read = (diag, key, default_value) => {
+let read = (pathname, default_value, diag) => {
if (diag)
- console.log('reading', key)
+ console.log('reading', pathname)
let result = {dirty: false}
result.done = (
async () => {
+ let text
try {
- result.value = JSON.parse(
- await fs_readFile(key, {encoding: 'utf-8'})
- )
+ text = await fs_readFile(pathname, {encoding: 'utf-8'})
}
catch (err) {
- if (
- default_value === undefined ||
- !(err instanceof Error) ||
- err.code !== 'ENOENT'
- )
- throw err
- result.value = default_value
+ if (!(err instanceof Error) || err.code !== 'ENOENT')
+ throw err;
+ try {
+ await fs_rename(pathname + '.temp', pathname)
+ text = await fs_readFile(pathname, {encoding: 'utf-8'})
+ }
+ catch (err) {
+ if (
+ default_value === undefined ||
+ !(err instanceof Error) ||
+ err.code !== 'ENOENT'
+ )
+ throw err;
+ }
}
+ result.value = text === undefined ? default_value : JSON.parse(text)
}
)()
return result
JSONCache.prototype.read = async function(key, default_value) {
let result = this.map.get(key)
if (result === undefined) {
- result = read(this.diag, key, default_value)
+ result = read(key, default_value, this.diag)
this.map.set(key, result)
await result.done
delete result.done
return result.value
}
-let write = (diag, key, result, timeout) => {
+let write = (pathname, result, timeout, diag) => {
if (!result.dirty) {
result.dirty = true
setTimeout(
async () => {
if (diag)
- console.log('writing', key)
+ console.log('writing', pathname)
result.dirty = false
+ let temp_pathname = pathname + '.temp'
let text = JSON.stringify(result.value) + '\n'
try {
- await fs_writeFile(key, text, {encoding: 'utf-8'})
+ await fs_writeFile(temp_pathname, text, {encoding: 'utf-8'})
+ try {
+ await fs_unlink(pathname)
+ }
+ catch (err) {
+ if (!(err instanceof Error) || err.code !== 'ENOENT')
+ throw err
+ }
+ await fs_rename(temp_pathname, pathname)
}
catch (err) {
console.err(err.stack || err.message)
await result.done
result.value = value
}
- write(this.diag, key, result, timeout)
+ write(key, result, timeout, this.diag)
}
JSONCache.prototype.modify = async function(
// atomically check that result.done === undefined before the modification
let result = this.map.get(key)
if (result === undefined) {
- result = read(this.diag, key, default_value)
+ result = read(key, default_value, this.diag)
this.map.set(key, result)
await result.done
delete result.done
result.done = (
async () => {
await modify_func(result)
- write(this.diag, key, result, timeout)
+ write(key, result, timeout, this.diag)
}
)()
await result.done
must be called with the `new` operator. The resulting cache object is intended
to store arbitrary node.js JSON objects, which are read from disk files and
modified (repeatedly) during the execution of your program. The cache tracks
-the on-disk path of the object, and writes it back to that path after a delay
+the on-disk pathname of the object, and writes it back to there after a delay
time. A simple form of locking is implemented to support atomic modifications.
## Calling API
}
```
+## About system crashes
+If the system crashes while writing the JSON file, a partially written file
+will unavoidably be left on the disk after the system reboots. To be robust
+against this situation, we write the modified JSON out to a temporary file
+first (whose pathname is the `key` value plus `'.temp'`), and then rename it
+into place. The only problem that can then happen is if the crash occurs after
+deleting the original but before renaming the temporary in its place. To guard
+against this, when opening the file we check for the requested file and then if
+that does not exist, we attempt to rename a temporary file in and then re-try.
-the `balances.json` modification inside the `transactions.json`
-modification. A consistent order must be chosen (in here, always acquiring the `bal
+We do not guarantee that atomic modifications spanning several files will be
+atomic across a system crash. The renaming system is only intended to guard
+against data loss. If desynchronization is an issue, then all files concerned
+should be scanned on system startup, and synchronization fixed up as necessary.
## About asynchronicity
## To be implemented
-At present, the modified JSON file is written over the previous one, and it is
-vulnerable to a system crash occuring leaving a partially written JSON file. In
-this case, data loss will occur, since the old contents of the file will not be
-accessible. We have designed a protocol to address this: the modified file is
-written to a new name, e.g. `example.json.tmp` instead of `example.json`, then
-the original is deleted and the new file renamed. If `example.json` does not
-exist at system startup, but `example.json.tmp` does, it means that a crash
-occurred, leaving a _fully written_ replacement file, but after deleting the
-original. In this case, it is safe to complete the renaming at system startup.
-We plan to implement this urgently, in the next version of the `json_cache`.
-
It is intended that we will shortly add a timer function (or possibly just a
function that the user should call periodically) to flush objects from the
cache after a stale time, on the assumption that the object might not be
public / json_cache_rw.git / commitdiff