Read EXIF data from images

main.js

@@ -18,13 +18,16 @@ function App() {
 }
 
 App.prototype.init = async function() {
-    require('./router/mdi.js').register(this);
+    [
+        "./router/mdi.js",
+        "./router/api.js"
+    ].forEach(i => require(i).register(this));
     await this.databaseHelper.init();
 }
 
 App.prototype.run = function() {
     http.createServer(this.router).listen(CONFIG.port);
-    this.libraryManager.updateLibraries(this.databaseHelper).then(() => process.exit());
+    this.libraryManager.updateLibraries(this.databaseHelper);
 }
 
 console.info = () => {};

model/mediaItemMeta.js

@@ -0,0 +1,42 @@
+
+const DatabaseModel = require("./DatabaseModel.js").DatabaseModel;
+
+function MediaFileMetaModel(md5sum, key, value) {
+    DatabaseModel.call(this);
+    this.md5sum = md5sum || "";
+    this.key = key || "";
+    this.value = value || "";
+}
+
+MediaFileMetaModel.prototype = Object.create(DatabaseModel.prototype);
+
+MediaFileMetaModel.prototype.getTableName = function() {
+    return "mediaMeta";
+}
+
+MediaFileMetaModel.prototype.createOrUpdateBase = async function(dbHelper) {
+    await dbHelper.runSql(`CREATE TABLE IF NOT EXISTS 'mediaMeta' (
+        md5sum STRING NOT NULL,
+        key varchar(32) NOT NULL,
+        value varchar(32) NOT NULL,
+        PRIMARY KEY (md5sum, key))`);
+}
+
+MediaFileMetaModel.prototype.describe = function() {
+    return {
+        "md5sum": this.md5sum,
+        "key": this.key,
+        "value": this.value
+    };
+}
+
+MediaFileMetaModel.prototype.versionColumn = function() { return ""; }
+
+MediaFileMetaModel.prototype.fromDb = function(dbObj) {
+    this.md5sum = dbObj["md5sum"];
+    this.key = dbObj["key"];
+    this.value = dbObj["value"];
+}
+
+module.exports.MediaFileMetaModel = MediaFileMetaModel;
+

model/mediaItemTag.js

@@ -0,0 +1,38 @@
+
+const DatabaseModel = require("./DatabaseModel.js").DatabaseModel;
+
+function MediaFileTagModel(md5sum, tag) {
+    DatabaseModel.call(this);
+    this.md5sum = md5sum || "";
+    this.tag = tag || "";
+}
+
+MediaFileTagModel.prototype = Object.create(DatabaseModel.prototype);
+
+MediaFileTagModel.prototype.getTableName = function() {
+    return "mediaTag";
+}
+
+MediaFileTagModel.prototype.createOrUpdateBase = async function(dbHelper) {
+    await dbHelper.runSql(`CREATE TABLE IF NOT EXISTS 'mediaTag' (
+        md5sum STRING NOT NULL,
+        tag varchar(32) NOT NULL,
+        PRIMARY KEY (md5sum, tag))`);
+}
+
+MediaFileTagModel.prototype.describe = function() {
+    return {
+        "md5sum": this.md5sum,
+        "tag": this.tag
+    };
+}
+
+MediaFileTagModel.prototype.versionColumn = function() { return ""; }
+
+MediaFileTagModel.prototype.fromDb = function(dbObj) {
+    this.md5sum = dbObj["md5sum"];
+    this.tag = dbObj["tag"];
+}
+
+module.exports.MediaFileTagModel = MediaFileTagModel;
+

package.json

@@ -14,6 +14,8 @@
   "license": "ISC",
   "dependencies": {
     "crypto": "^1.0.1",
+    "geokit": "^1.1.0",
+    "imagemagick": "^0.1.3",
     "ldap": "^0.7.1",
     "ldapjs": "^3.0.4",
     "mime-types": "^2.1.35",

router/api.js

@@ -0,0 +1,13 @@
+
+const fs = require('fs');
+const CONFIG = require('../src/config.js');
+
+module.exports = { register: app => {
+    app.router.get("/api/media/list", async (req, res) => {
+        app.routerUtils.onRequest(req);
+        if (app.routerUtils.apiRequireLogin(req, res))
+            return;
+        app.routerUtils.jsonResponse(res, data);
+    });
+}};
+

src/databaseHelper.js

@@ -2,9 +2,11 @@
 const sqlite3 = require('sqlite3');
 const SessionModel = require('../model/session.js').SessionModel;
 const MediaFileModel = require('../model/mediaItem.js').MediaFileModel;
+const MediaMetaModel = require('../model/mediaItemMeta.js').MediaFileMetaModel;
+const MediaTagModel = require('../model/mediaItemTag.js').MediaFileTagModel;
 const CONFIG = require('./config.js');
 
-let ALL_MODELS = [ SessionModel, MediaFileModel ];
+let ALL_MODELS = [ SessionModel, MediaFileModel, MediaMetaModel, MediaTagModel ];
 
 function DatabaseHelper() {
     this.db = null;
@@ -66,6 +68,27 @@ DatabaseHelper.prototype.insertOne = async function(object) {
     await this.runSql(request, args);
 }
 
+DatabaseHelper.prototype.insertMultipleSameTable = async function(arr) {
+    let columns = [];
+    let args = [];
+
+    if (!arr || !arr.length)
+        return;
+    for (let [key, val] of Object.entries(arr[0].describe()))
+        columns.push("`"+key+"`");
+    let request = "insert into `" +arr[0].getTableName()
+            +"` (" +columns.join(", ")
+            +") VALUES ";
+    let valuesArr = [];
+    request += arr.map(i => ("(" +columns.map(i => "?").join(",")+")")).join(',');
+    for (let i of arr)
+    {
+        for (let [key, val] of Object.entries(i.describe()))
+            args.push(val);
+    }
+    await this.runSql(request, args);
+}
+
 DatabaseHelper.prototype.upsertOne = async function(obj) {
     let columns = [];
     let columnsVal = [];

src/fileTypeManager.js

@@ -1,12 +1,16 @@
 
 const parsers = [
-    require ('./filetype/jpeg.js')
+    require ('./filetype/imagemagick.js'),
+    require ('./filetype/meta.js')
 ];
 
-module.exports.createMeta = async function(path) {
+module.exports.createMeta = async function(fileObj) {
     let result = {};
-    for (let i of await Promise.all(parsers.map(i => i.parse(path))))
-        i && Object.assign(result, i);
+    for (let i of await Promise.allSettled(parsers.map(i => i.parse(fileObj))))
+    {
+        i.value && Object.assign(result, i.value);
+        i.reason && console.error("Failed to parse file ", fileObj.path, i.reason);
+    }
     return result;
 }
 

src/filetype/imagemagick.js

@@ -0,0 +1,117 @@
+
+const im = require('imagemagick');
+const geokit = require('geokit');
+
+function readMeta(path) {
+    return new Promise((ok, ko) => {
+        im.identify(['-format', '%[EXIF:*]Compression=%[compression]\nWidth=%w\nHeight=%h\n', path], function(err, stdout)
+        {
+            if (err)
+                return ok(null);
+            var meta = {};
+            for (const line of stdout.split(/\n/))
+            {
+                var eq_p = line.indexOf('=');
+                if (eq_p === -1)
+                    continue;
+                var key = line.substr(0, eq_p).replace('/','-'),
+                    value = line.substr(eq_p+1).trim();
+                var p = key.indexOf(':');
+                if (p !== -1)
+                    key = key.substr(p+1);
+                key = key.charAt(0).toLowerCase() + key.slice(1);
+                meta[key] = value;
+            }
+            ok(meta);
+        });
+    });
+}
+
+function exifDate(value) {
+    if (!value)
+        return undefined;
+    value = value.split(/ /);
+    return new Date(value[0].replace(/:/g, '-')+' '+value[1]+' +0000');
+}
+
+function exifSlash(value) {
+    if (!value)
+        return undefined;
+    if (!(/[0-9]+\/[0-9]+/).test(value))
+        return value;
+    return eval(value);
+}
+
+function ExifGps(data) {
+    /*
+     *    gPSLatitude: '46/1,54/1,3500/100',
+          gPSLatitudeRef: 'N',
+          gPSLongitude: '23/1,48/1,3614/100',
+          gPSLongitudeRef: 'E',
+    */
+    this.data = null;
+    if (!data.gPSLatitude || !data.gPSLongitude || !(/^[0-9\-,\/]+$/).test(data.gPSLatitude) || !(/^[0-9\-,\/]+$/).test(data.gPSLongitude))
+        return;
+    try
+    {
+        this.data = {
+            lat: data.gPSLatitude.split(',').map(eval),
+            lon: data.gPSLongitude.split(',').map(eval)
+        };
+    }
+    catch(err)
+    {
+        this.data = null;
+        return;
+    }
+    if (data.gPSLatitudeRef && data.gPSLatitudeRef !== 'N' && this.data.lat[0] >= 0)
+        this.data.lat[0] *= -1;
+    if (data.gPSLongitudeRef && data.gPSLongitudeRef !== 'E' && this.data.lon[0] >= 0)
+        this.data.lon[0] *= -1;
+    this.data.lat = this.data.lat[0] + this.data.lat[1] / 60 + this.data.lat[2]/3600;
+    this.data.lon = this.data.lon[0] + this.data.lon[1] / 60 + this.data.lon[2]/3600;
+    if (!this.data.lat || isNaN(this.data.lat) || !this.data.lon || isNaN(this.data.lon))
+        this.data = null;
+}
+
+ExifGps.prototype.toGps = function() {
+    if (!this.data)
+        return undefined;
+    return JSON.stringify([this.data.lat, this.data.lon]);
+}
+
+ExifGps.prototype.toGeoHash = function() {
+    if (!this.data)
+        return undefined;
+    return geokit.hash({lat: this.data.lat, lng: this.data.lon});
+}
+
+module.exports.parse = async (fileObj) => {
+    if (!fileObj.mimeType.startsWith('image/'))
+        return {};
+    let imdata = await readMeta(fileObj.path);
+    if (!imdata)
+        return {};
+    let result = {};
+    result.artist = imdata.artist || undefined;
+    result.exposureTime = exifSlash(imdata.exposureTime);
+    result.exposureTimeStr = imdata.exposureTime || undefined;
+    result.dateTime = exifDate(imdata.dateTimeDigitized || imdata.dateTimeOriginal);
+    result.fNumber = exifSlash(imdata.fNumber);
+    result.focal = exifSlash(imdata.focalLength);
+    result.lensModel = imdata.lensModel || undefined;
+    result.camera = ((imdata.model || "") + (imdata.model && imdata.make ? " " : "") + (imdata.make || "")) || "";
+    result.software = imdata.software || undefined;
+    result.iso = Number.parseInt(imdata.photographicSensitivity) || undefined;
+    result.width = imdata.width || undefined;
+    result.height = imdata.height || undefined;
+    result.compression = imdata.compression || undefined;
+    const gpsData = new ExifGps(imdata);
+    result.gpsLocation = gpsData.toGps();
+    result.geoHash = gpsData.toGeoHash();
+    for (let i of Object.keys(result))
+        if (result[i] === undefined || result[i].length === 0)
+            delete result[i];
+    return result;
+}
+

src/filetype/jpeg.js

@@ -1,5 +0,0 @@
-
-module.exports.parse = (path) => {
-    return {};
-}
-

src/filetype/meta.js

@@ -1,15 +1,9 @@
 
-function Meta() {
-    this.height = 0;
-    this.width = 0;
-    this.snapTime = new Date(0);
-    this.cameraBrand = "";
-    this.cameraModel = "";
-    this.objBrand = "";
-    this.objModel = "";
-    this.fNumber = 0;
-    this.delay = 0;
-}
+const fs = require('fs');
 
-module.exports.Meta = Meta;
+module.exports.parse = async (fileObj) => {
+    return {
+        fileSize: (await fs.promises.stat(fileObj.path)).size
+    };
+}
 

src/library.js

@@ -5,6 +5,7 @@ const md5Stats = require('./md5sum.js').stats;
 const FileTypeManager = require('./fileTypeManager.js');
 
 const MediaFileModel = require("../model/mediaItem.js").MediaFileModel;
+const MediaFileMetaModel = require("../model/mediaItemMeta.js").MediaFileMetaModel;
 const MANAGED_FILES = [ ".cr2" ]; // TODO
 const BUFFER_MAXSIZE = 100;
 
@@ -21,19 +22,39 @@ File.prototype.enrich = async function() {
     this.mimeType = mime.lookup(this.name) || "";
     const lowerName = this.name.toLowerCase();
     this.isMedia = this.mimeType.startsWith("image/") || this.mimeType.startsWith("video/") || !!MANAGED_FILES.find(i => lowerName.endsWith(i));
-    this.checksum = this.isMedia ? await md5Stats(this.path) : "";
-    this.meta = await FileTypeManager.createMeta(this.path);
+    this.checksum = "";
+    this.meta = null;
+    if (this.isMedia)
+    {
+        this.meta = await FileTypeManager.createMeta(this);
+        this.checksum = await md5Stats(this.path);
+    }
+}
+
+File.prototype.saveDb = async function(db) {
+    let entity = new MediaFileModel();
+    let _this = this;
+    entity.path = this.path;
+    entity.md5sum = this.checksum;
+    await db.insertOne(entity);
+    this.meta.photochamberImport = new Date();
+    let metaEntities = Object.keys(this.meta).map(i => new MediaFileMetaModel(_this.checksum, i, _this.meta[i]));
+    await db.insertMultipleSameTable(metaEntities);
+}
+
+async function enrichAll(lib) {
+    for (let i =0; i < lib.buff.length; i += 5)
+        await Promise.allSettled(lib.buff.slice(i, i+5).map(i => i.enrich()));
 }
 
 async function Library_doUpdate(dbHelper, lib) {
     if (lib.buff.length === 0)
         return;
-    const dbItems = await dbHelper.fetchRaw("path", MediaFileModel.prototype.getTableName.call(null), { "path": lib.buff.map(i => i.path) });
+    const dbItems = (await dbHelper.fetchRaw("path", MediaFileModel.prototype.getTableName.call(null), { "path": lib.buff.map(i => i.path) })).map(i => i.path);
     lib.buff = lib.buff.filter(i => dbItems.indexOf(i.path) === -1);
-    console.log(lib.buff);
-    await Promise.allSettled(lib.buff.map(i => i.enrich()));
+    await enrichAll(lib);
     lib.buff = lib.buff.filter(i => !!i.checksum);
-    // TODO update
+    await Promise.allSettled(lib.buff.map(i => i.saveDb(dbHelper)));
     lib.foundMedias = lib.foundMedias.concat(lib.buff);
     lib.buff = [];
 }

src/libraryManager.js

@@ -4,6 +4,7 @@ const Library = require('./library.js').Library;
 
 function LibraryManager() {
     this.libraries = [];
+    this.updating = false;
 
     for (let i of CONFIG.photoLibraries)
         this.push(i);
@@ -11,11 +12,19 @@ function LibraryManager() {
 
 LibraryManager.prototype.push = function(path) {
     this.libraries.push(new Library(path));
+    // FIXME remove from db file that does not exists any more
 }
 
-LibraryManager.prototype.updateLibraries = function(dbHelper) {
-    return Promise.allSettled(this.libraries.map(i => i.updateLibrary(dbHelper)));
+LibraryManager.prototype.updateLibraries = async function(dbHelper) {
+    if (this.updating)
+        return;
+    this.updating = true;
+    await Promise.allSettled(this.libraries.map(i => i.updateLibrary(dbHelper)));
+    this.updating = false;
 }
 
+LibraryManager.prototype.isUpdating = function()
+{ return this.updating; }
+
 module.exports.LibraryManager = new LibraryManager();