## Overview
-The `disk_build` package exports a function `disk_build(pathname, build_func)`,
-which given a pathname to an existing source file, will generate a a pathname
-for a corresponding on-disk temporary file, check whether this exists and is
-newer than the original, and rebuild it via a caller-provided function if not.
+The `disk_build` package exports a function
+`disk_build(pathname, build_func, diag)`,
+which given a pathname to an existing source file, will generate a pathname for
+a corresponding on-disk output file, check whether this exists and is newer
+than the original, and if not, rebuild it via a caller-provided function.
## Calling API
The interface for the `disk_build`-provided helper function `disk_build()` is:
-`await disk_build(pathname, build_func)` — generates the corresponding
-`built_pathname`, checks if the temporary file `built_pathname` exists and is
-newer than `pathname`, and if not, calls the user-provided callback function
-`build_func(built_pathname)` to rebuild the temporary file before returning.
+`await disk_build(pathname, build_func, diag)` — checks if the source
+file given in `pathname` exists, generates the corresponding output pathname
+and dependency-file pathname, reads the dependency file if possible, checks the
+output is up-to-date with respect to all dependencies, and if not, generates
+a temporary output pathname `temp_pathname`, calls the user-provided callback
+function `build_func(temp_pathname)` to build the output into the temporary,
+possibly writes a new dependency file, then renames everything into place.
+
+Finally, it returns an dictionary `{deps: ..., pathname: ...}` where `deps` is
+a list of dependencies not including the original, and `pathname` is the output
+file; these might come from the existing disk files or newly written ones.
The interface for the user-provided callback function `build_func()` is:
-`await build_func(text, built_pathname)` — user must create the file
-`built_pathname` and then return, or alternatively an exception can be thrown.
+`await build_func(temp_pathname)` — the user must create the file
+`temp_pathname` and then return, or alternatively an exception can be thrown.
+The return value can be either `undefined` if no dependency file should be
+written, or a list of pathnames referred to (not including the original).
+
+## About asynchronicity
+
+There is nothing too complicated here, both `disk_build()` and `build_func()`
+are asynchronous and thus return a `Promise`, and when the inner `Promise`
+resolves, the outer `Promise` resolves in turn. Either level throws exceptions,
+with various causes such as disk errors, file not found, syntax error, etc.
+
+There is no protection against multiple clients trying to access the same file
+at once while the asychronous building operation proceeds. This is because the
+accesses to `disk_build()` are supposed to be wrapped in a `BuildCache`, which
+will exclude multiple access. There was no point making `disk_build()` keep a
+dictionary of operations in progress when `BuildCache` already does this, since
+`disk_build()` does not otherwise need any state kept in RAM (it has the disk).
+
+## Usage example
+
+Simple usage, showing use of the `clean-css` CSS minimizer with `disk_build()`,
+in such a way as to return a string containing the minified CSS from the given
+input file described by `pathname`, without re-converting if already converted:
+```
+let CleanCSS = require('clean-css')
+let disk_build = require('@ndcode/disk_build')
+let fs = require('fs')
+let util = require('util')
+
+let clean_css = new CleanCSS({returnPromise: true})
+let fs_readFile = util.promisify(fs.readFile)
+let fs_writeFile = util.promisify(fs.writeFile)
+
+let get_css_min = async pathname => {
+ let render = await disk_build(
+ pathname,
+ async temp_pathname => {
+ let render = await clean_css.minify(
+ await fs_readFile(pathname, {encoding: 'utf-8'})
+ )
+ return /*await*/ fs_writeFile(
+ temp_pathname,
+ render.styles,
+ {encoding: 'utf-8'}
+ )
+ },
+ true // diagnostics on
+ )
+ return /*await*/ fs_readFile(render.pathname)
+}
+```
+Since it is rather common that various minimizers and template renderers return
+an dictionary containing the result of processing plus some other information,
+and so does `disk_cache()`, we've adopted the convention of putting the result
+of such calls in a variable called `render`, despite multiple use of the name.
+
+A more complicated example using a `BuildCache` to exclude multiple callers:
+```
+let BuildCache = require('@ndcode/build_cache')
+let CleanCSS = require('clean-css')
+let disk_build = require('@ndcode/disk_build')
+let fs = require('fs')
+let util = require('util')
+
+let build_cache = new BuildCache(true) // diagnostics on
+let clean_css = new CleanCSS({returnPromise: true})
+let fs_readFile = util.promisify(fs.readFile)
+let fs_writeFile = util.promisify(fs.writeFile)
+
+let get_css_min = pathname => /*await*/ build_cache.get(
+ pathname,
+ async result => {
+ let render = await disk_build(
+ pathname,
+ async temp_pathname => {
+ let render = await clean_css.minify(
+ await fs_readFile(pathname, {encoding: 'utf-8'})
+ )
+ return /*await*/ fs_writeFile(
+ temp_pathname,
+ render.styles,
+ {encoding: 'utf-8'}
+ )
+ },
+ true // diagnostics on
+ )
+ result.value = /*await*/ fs_readFile(render.pathname)
+ }
+}
+```
+In the above there was no dependency tracking needed or wanted, since CSS files
+cannot include further CSS source files. `Less` files can, handled as follows:
+```
+let BuildCache = require('@ndcode/build_cache')
+let disk_build = require('@ndcode/disk_build')
+let fs = require('fs')
+let util = require('util')
+let less = require('less/lib/less-node')
+let path = require('path')
+
+let build_cache = new BuildCache(true) // diagnostics on
+let fs_readFile = util.promisify(fs.readFile)
+let fs_writeFile = util.promisify(fs.writeFile)
+
+let get_css_less = pathname => /*await*/ build_cache.get(
+ pathname,
+ async result => {
+ let render = await disk_build(
+ pathname,
+ async temp_pathname => {
+ let render = await less.render(
+ await fs_readFile(pathname, {encoding: 'utf-8'}),
+ {
+ compress: true,
+ filename: pathname,
+ pathnames: [path.posix.dirname(pathname)]
+ }
+ )
+ await fs_writeFile(
+ temp_pathname,
+ render.css,
+ {encoding: 'utf-8'}
+ )
+ return render.imports
+ },
+ true // diagnostics on
+ )
+ result.value = /*await*/ fs_readFile(render.pathname)
+ }
+}
+```
+We are relying on `fs_readFile()` or `less.render()` to throw exceptions if the
+original stylesheet or any included stylesheet is not found or contains errors.
+
+Also, note how much simplified the handling of asynchronicity is when using the
+ES7 `async`/`await` syntax. We recommend to do this for all new code, and to do
+it consistently, even if the use of `Promise` directly might give shorter code.
+
+Code which is not already promisified, can be promisified as shown, or else we
+can add specific conversions in places by code like: `await new Promise(...)`.
+Note the comment `/* await */` where a `Promise` is passed through from a lower
+level routine, an explicit `await` would be consistent here but less efficient.
## About diagnostics
-A diagnostic will be printed if the temporary file is up-to-date and does not
-need to be rebuilt. No diagnostic is printed if the `build_func()` callback is
-invoked, since it is expected that the user can provide an appropriate one
-(allowing the diagnostic to be suppressed if the source file isn't readable).
-
-## To be implemented
-
-It is intended that dependencies will be tracked in the future, so that an
-on-disk temporary file can be built from several on-disk sources. In that case
-we will write a second temporary file `*.deps` containing a list of the build
-dependencies, so that up-to-dateness can be checked from only the disk status.
+If `diag` is passed as `true`, then a diagnostic will be printed indicating
+whether an output file is being rebuilt, or an existing output is being reused.
## GIT repository
let path = require('path')
let util = require('util')
+let fs_readFile = util.promisify(fs.readFile)
+let fs_rename = util.promisify(fs.rename)
let fs_stat = util.promisify(fs.stat)
+let fs_unlink = util.promisify(fs.unlink)
+let fs_writeFile = util.promisify(fs.writeFile)
-let disk_build = async (pathname, build_func) => {
+let disk_build = async (pathname, build_func, diag) => {
let stats = await fs_stat(pathname)
- let built_pathname = path.posix.resolve(
+ let out_pathname = path.posix.resolve(
path.posix.dirname(pathname),
`.${path.posix.basename(pathname)}`
)
+ let out_deps_pathname = out_pathname + '.deps'
- let built_stats
+ let out_deps
try {
- built_stats = await fs_stat(built_pathname)
+ out_deps = JSON.parse(
+ await fs_readFile(out_deps_pathname, {encoding: 'utf-8'})
+ )
}
catch (err) {
if (!(err instanceof Error) || err.code !== 'ENOENT')
throw err
- //built_stats = undefined
+ out_deps = []
}
- if (built_stats === undefined || stats.mtimeMs > built_stats.mtimeMs)
- await build_func(built_pathname)
- else
- console.log('reloading', built_pathname)
- return built_pathname
+ let rebuild = false
+ try {
+ let out_stats = await fs_stat(out_pathname)
+ if (stats.mtimeMs > out_stats.mtimeMs)
+ rebuild = true
+ else
+ for (let i = 0; i < out_deps.length; ++i)
+ if ((await fs_stat(out_deps[i])).mtimeMs > out_stats.mtimeMs) {
+ rebuild = true
+ break
+ }
+ }
+ catch (err) {
+ if (!(err instanceof Error) || err.code !== 'ENOENT')
+ throw err
+ rebuild = true
+ }
+
+ if (rebuild) {
+ if (diag)
+ console.log('building', out_pathname)
+ let out_temp_pathname = out_pathname + '.temp'
+ out_deps = await build_func(out_temp_pathname)
+ try {
+ await fs_unlink(out_pathname)
+ }
+ catch (err) {
+ if (!(err instanceof Error) || err.code !== 'ENOENT')
+ throw err
+ }
+ try {
+ await fs_unlink(out_deps_pathname)
+ }
+ catch (err) {
+ if (!(err instanceof Error) || err.code !== 'ENOENT')
+ throw err
+ }
+ if (out_deps === undefined)
+ out_deps = []
+ else {
+ let out_deps_temp_pathname = out_deps_pathname + '.temp'
+ await fs_writeFile(
+ out_deps_temp_pathname,
+ JSON.stringify(out_deps) + '\n',
+ {encoding: 'utf-8'}
+ )
+ await fs_rename(out_deps_temp_pathname, out_deps_pathname)
+ }
+ await fs_rename(out_temp_pathname, out_pathname)
+ }
+ else if (diag)
+ console.log('reloading', out_pathname)
+
+ return {deps: out_deps, pathname: out_pathname}
}
module.exports = disk_build
public / disk_build.git / commitdiff