Make analytics.js use json_cache, add analytics diagnostics and fix some bugs

diff --git a/analytics.js b/analytics.js
index bdec882..09bc95a 100644 (file)
--- a/analytics.js
+++ b/analytics.js
@@ -1,7 +1,5 @@
 let fs = require('fs')
-let util = require('util')
-
-let fs_writeFile = util.promisify(fs.writeFile)
+let JSONCache = require('json_cache')
 
 try {
   fs.mkdirSync('analytics')
@@ -11,79 +9,25 @@ catch (err) {
     throw err
 }
 
-let sessions
-{
-  let text 
-  try {
-    text = fs.readFileSync('analytics/sessions.json', {encoding: 'utf-8'})
-  }
-  catch (err) {
-    if (err.code !== 'ENOENT') // should check error type
-      throw err
-    text = '{}'
-  }
-  sessions = JSON.parse(text)
-}
-let sessions_is_dirty = false
-let sessions_dirty = () => {
-  if (!sessions_is_dirty) {
-    sessions_is_dirty = true
-    setTimeout(sessions_write, 5000)
-  }
-}
-let sessions_write = async () => {
-  try {
-    sessions_is_dirty = false;
-    console.log('writing analytics/sessions.json')
-    await fs_writeFile(
-      'analytics/sessions.json',
-      JSON.stringify(sessions),
-      {encoding: 'utf-8'}
-    )
-  }
-  catch (err) {
-    console.error(err.stack || err.message)
-  }
-}
+let json_cache = new JSONCache()
+json_cache.diag = true
 
+let sessions
 let pageviews
-{
-  let text 
-  try {
-    text = fs.readFileSync('analytics/pageviews.json', {encoding: 'utf-8'})
-  }
-  catch (err) {
-    if (err.code !== 'ENOENT') // should check error type
-      throw err
-    text = '{}'
-  }
-  pageviews = JSON.parse(text)
-}
-let pageviews_is_dirty = false
-let pageviews_dirty = () => {
-  if (!pageviews_is_dirty) {
-    pageviews_is_dirty = true
-    setTimeout(pageviews_write, 5000)
-  }
-}
-let pageviews_write = async () => {
-  try {
-    pageviews_is_dirty = false;
-    console.log('writing analytics/pageviews.json')
-    await fs_writeFile(
-      'analytics/pageviews.json',
-      JSON.stringify(pageviews),
-      {encoding: 'utf-8'}
-    )
-  }
-  catch (err) {
-    console.error(err.stack || err.message)
-  }
+let refresh = async () => {
+  sessions = await json_cache.get('analytics/sessions.json', {})
+  pageviews = await json_cache.get('analytics/pageviews.json', {})
+
+  // a bit awkward... changing the exports on the fly
+  exports.sessions = sessions
+  exports.pageviews = pageviews
 }
+let sessions_dirty = () => json_cache.set('analytics/sessions.json')
+let pageviews_dirty = () => json_cache.set('analytics/pageviews.json')
 
+exports.json_cache = json_cache
 exports.sessions = sessions
-exports.sessions_dirty = sessions_dirty
-exports.sessions_write = sessions_write
 exports.pageviews = pageviews
+exports.refresh = refresh
+exports.sessions_dirty = sessions_dirty
 exports.pageviews_dirty = pageviews_dirty
-exports.pageviews_write = pageviews_write

diff --git a/package.json b/package.json
index 47b1492..5cabaf3 100644 (file)
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
     "fs": "^0.0.1-security",
     "http": "^0.0.0",
     "https": "^1.0.0",
+    "json_cache": "file:../json_cache.git/json_cache-1.0.0.tgz",
     "jstemplate": "file:../jstemplate.git/jstemplate-3.4.9.tgz",
     "less": "^3.8.1",
     "querystring": "^0.2.0",

diff --git a/resources.js b/resources.js
index e5105b7..42f49a3 100644 (file)
--- a/resources.js
+++ b/resources.js
@@ -11,7 +11,7 @@ let fs_readFile = util.promisify(fs.readFile)
 let yauzl_open = util.promisify(yauzl.open)
 
 let build_cache_js = new BuildCache()
-let get_js = async path => await build_cache_js.get(
+let get_js = path => build_cache_js.get(
   path,
   async result => {
     console.log('compiling', path)
@@ -20,7 +20,7 @@ let get_js = async path => await build_cache_js.get(
 )
 
 let build_cache_json = new BuildCache()
-let get_json = async path => await build_cache_json.get(
+let get_json = path => build_cache_json.get(
   path,
   async result => {
     console.log('parsing', path)
@@ -29,7 +29,7 @@ let get_json = async path => await build_cache_json.get(
 )
 
 let build_cache_less = new BuildCache()
-let get_less = async (path, site_root, dir_name) => await build_cache_less.get(
+let get_less = (path, site_root, dir_name) => build_cache_less.get(
   path,
   async result => {
     console.log('compiling', path)
@@ -63,7 +63,7 @@ let get_less = async (path, site_root, dir_name) => await build_cache_less.get(
 )
 
 let build_cache_text = new BuildCache()
-let get_text = async path => await build_cache_text.get(
+let get_text = path => build_cache_text.get(
   path,
   async result => {
     console.log('reading', path)
@@ -72,7 +72,7 @@ let get_text = async path => await build_cache_text.get(
 )
 
 let build_cache_zip = new BuildCache()
-let get_zip = async path => await build_cache_zip.get(
+let get_zip = path => build_cache_zip.get(
   path,
   async result => {
     console.log('decompressing', path)
@@ -117,7 +117,7 @@ let get_zip = async path => await build_cache_zip.get(
 )
 
 let build_cache_zet = new BuildCache()
-let get_zet = async path => await build_cache_zet.get(
+let get_zet = path => build_cache_zet.get(
   path,
   async result => {
     console.log('opening', path)

diff --git a/site.js b/site.js
index c842203..d65c934 100644 (file)
--- a/site.js
+++ b/site.js
@@ -43,7 +43,7 @@ let app = async (site, site_root, req, res, protocol) => {
       if (err.code !== 'ENOENT')
         throw err
       if (!dir_name_is_pub) {
-        temp = dir_name + '.pub'
+        let temp = dir_name + '.pub'
         try {
           stats = await fs_stat(site_root + temp)
           dir_name = temp
@@ -75,7 +75,7 @@ let app = async (site, site_root, req, res, protocol) => {
   }
   let page = path.slice(1).join('/')
 
-  temp = file_name.lastIndexOf('.')
+  let temp = file_name.lastIndexOf('.')
   let file_type = temp === -1 ? '' : file_name.substring(temp + 1)
   let mime_type =
     config.mime_types.hasOwnProperty(file_type) ?
@@ -83,8 +83,12 @@ let app = async (site, site_root, req, res, protocol) => {
     config.mime_type_default
 
   if (file_type == 'html') {
-    if (!analytics.sessions.hasOwnProperty(site))
+    await analytics.refresh()
+
+    if (!analytics.sessions.hasOwnProperty(site)) {
+      console.log('new site_sessions', site)
       analytics.sessions[site] = {}
+    }
     let site_sessions = analytics.sessions[site]
     let cookies = cookie.parse(req.headers.cookie || ''), session_key
     if (
@@ -92,6 +96,7 @@ let app = async (site, site_root, req, res, protocol) => {
       !site_sessions.hasOwnProperty(session_key = cookies.session_key)
     ) {
       session_key = crypto.randomBytes(16).toString('hex')
+      console.log('new session', session_key)
       site_sessions[session_key] = {}
     }
     let session = site_sessions[session_key]
@@ -105,26 +110,30 @@ let app = async (site, site_root, req, res, protocol) => {
     )
     session.expires = expires
 
-    if (!analytics.pageviews.hasOwnProperty(site))
+    if (!analytics.pageviews.hasOwnProperty(site)) {
+      console.log('new site_pageviews', site)
       analytics.pageviews[site] = {}
+    }
     let site_pageviews = analytics.pageviews[site]
-    if (!site_pageviews.hasOwnProperty(page))
+    if (!site_pageviews.hasOwnProperty(page)) {
+      console.log('new pageview', page)
       site_pageviews[page] = {visits: 0, unique_visits: 0}
+    }
     let pageview = site_pageviews[page]
     ++pageview.visits;
 
-  
-    if (!session.hasOwnProperty('analytics.pageviews'))
+    if (!session.hasOwnProperty('pageviews'))
       session.pageviews = {}
     let session_pageviews = session.pageviews
     if (!session_pageviews.hasOwnProperty(page)) {
+      console.log('new session_pageview', page)
       session_pageviews[page] = 0
       ++pageview.unique_visits
     }
     ++session_pageviews[page]
 
-    analytics.sessions_dirty()
-    analytics.pageviews_dirty()
+    await analytics.sessions_dirty()
+    await analytics.pageviews_dirty()
   }
 
   /*let*/ page = dir_name + '/' + file_name; let data
@@ -188,19 +197,19 @@ let app = async (site, site_root, req, res, protocol) => {
           ) + str, result
           switch (type) {
           case undefined:
-            result = await (await resources.get_js(path))(env, out, req)
+            result = /*await*/ (await resources.get_js(path))(env, out, req)
             break
           case 'js':
-            result = await resources.get_js(path)
+            result = /*await*/ resources.get_js(path)
             break
           case 'json':
-            result = await resources.get_json(path)
+            result = /*await*/ resources.get_json(path)
             break
           case 'text':
-            result = await resources.get_text(path)
+            result = /*await*/ resources.get_text(path)
             break
           case 'zet':
-            result = await resources.get_zet(path)
+            result = /*await*/ resources.get_zet(path)
             break
           default:
             assert(false)