photochamber-webserver/src/library.js

const fs = require('fs');
const path = require('path');
const mime = require("mime-types");
const md5Stats = require('craftlabhttpserver/src/md5sum').stats;
const md5String = require('craftlabhttpserver/src/md5sum.js').string;
const FileTypeManager = require('./fileTypeManager.js');

const { ACCESS_TO, AccessModel } = require("../model/access.js");
const MediaFileModel = require("../model/mediaItem.js").MediaFileModel;
const MediaFileMetaModel = require("../model/mediaItemMeta.js").MediaFileMetaModel;
const MediaFileTagModel = require("../model/mediaItemTag.js").MediaFileTagModel;
const MANAGED_FILES = [ /* ".cr2" */ ]; // TODO
const BUFFER_MAXSIZE = 100;

function File(fullPath, name) {
    this.name = name;
    this.path = fullPath;
    this.fixedSum = null;
    this.checksum = null;
    this.mimeType = null;
    this.isMedia = null;
    this.meta = {};
    this.dbItem = null;
    this.tags = [];
}

File.prototype.getIsMedia = function() {
    if (this.isMedia !== null)
        return this.isMedia;
    if (!this.mimeType)
        this.mimeType = mime.lookup(this.name) || "";
    this.isMedia = this.mimeType.startsWith("image/") || /*this.mimeType.startsWith("video/") || */!!MANAGED_FILES.find(i => this.name.toLowerCase().endsWith(i));
    return this.isMedia;
}

File.prototype.computeChecksum = async function() {
    if (!this.checksum && this.getIsMedia())
        this.checksum = await md5Stats(this.path);
    if (!this.fixedSum)
        this.fixedSum = this.checksum;
    return this.checksum;
}

File.prototype.enrich = async function() {
    this.mimeType = mime.lookup(this.name) || "";
    const lowerName = this.name.toLowerCase();
    this.meta = null;
    await this.computeChecksum();
    if (this.getIsMedia()) {
        this.meta = await FileTypeManager.createMeta(this);
        this.tags = this.meta.tags || [];
        this.meta.tags = undefined;
    }
}

File.prototype.saveDb = async function(db, libraryHash) {
    if (this.dbItem) {
        this.fixedSum = this.dbItem.fixedSum;
        await db.remove(MediaFileModel, { path: this.dbItem.path, fixedSum: this.dbItem.fixedSum });
        await db.remove(MediaFileMetaModel, { md5sum: this.dbItem.fixedSum, fromFile: true });
        await db.remove(MediaFileTagModel, { md5sum: this.dbItem.fixedSum, fromMeta: true });
    }
    let entity = new MediaFileModel();
    let _this = this;
    entity.path = this.path;
    entity.md5sum = await this.computeChecksum();
    entity.fixedSum = this.fixedSum;
    entity.date = this.meta.dateTime || new Date(fs.statSync(this.path)?.birthtimeMs || Date.now());
    await db.insertOne(entity);
    this.meta.photochamberImport = new Date();
    this.meta.libraryPath = libraryHash;
    let metaEntities = Object.keys(this.meta).filter(i => i !== 'tags').map(i => new MediaFileMetaModel(_this.fixedSum, i, _this.meta[i], true));
    await db.insertMultipleSameTable(metaEntities);
    if (this.tags.length) {
        let tagsEntities = this.tags.map(i => new MediaFileTagModel(_this.fixedSum, i, true));
        await db.insertMultipleSameTable(tagsEntities);
    }
    this.dbItem = null;
}

File.prototype.createThumbnail = async function(w, h, quality) {
    return await FileTypeManager.createThumbnail(this, w, h, quality);
}

async function enrichAll(lib) {
    for (let i =0; i < lib.buff.length; i += 5)
        await Promise.allSettled(lib.buff.slice(i, i+5).map(i => i.enrich()));
}

async function Library_doUpdate(dbHelper, lib) {
    lib.buff = lib.buff.filter(i => !!i.checksum);
    if (lib.buff.length === 0)
        return;
    const dbItems = (await dbHelper.fetchRaw(["path", "md5sum", "fixedSum"], MediaFileModel.prototype.getTableName.call(null), { "path": lib.buff.map(i => i.path) }));
    lib.buff.forEach(i => i.dbItem = dbItems.find(x => x.path == i.path));
    lib.buff = lib.buff.filter(i => !i.dbItem || i.dbItem.md5sum != i.checksum);
    await enrichAll(lib);
    (await Promise.allSettled(lib.buff.map(i => i.saveDb(dbHelper, lib.dbHash)))).forEach(x => { if (x.status === 'rejected') { console.log(`Cannot update item: `, x.reason); }});
    lib.foundMedias = lib.foundMedias.concat(lib.buff);
    lib.buff.length && console.log(`Updated ${lib.buff.length} media items`);
    lib.buff = [];
}

async function Library_depthupdate(dbHelper, library, dir) {
    for (let o of fs.readdirSync(dir, { withFileTypes: true })) {
        const fullPath = path.join(dir, o.name);
        if (o.isDirectory())
            await Library_depthupdate(dbHelper, library, fullPath);
        else if (o.isFile()) {
            let f = new File(fullPath, o.name);
            if (!f.getIsMedia())
                continue;
            await f.computeChecksum();
            library.buff.push(f);
            if (library.buff.length >= BUFFER_MAXSIZE)
                await Library_doUpdate(dbHelper, library);
        }
    }
}

function Library_isValid(_this) {
    if (fs.existsSync(_this.path))
        return true;
    console.error(`Library folder not found (${_this.path})`);
}

function Library(p) {
    this.path = path.normalize(p);
    this.dbHash = md5String(this.path);
}

Library.prototype.updateLibrary = async function(dbHelper) {
    const startTime = Date.now();
    console.log(`Starting update of library ${this.path}`);
    this.foundMedias = [];
    this.buff = [];
    this.errors = {};
    try {
        if (Library_isValid(this))
            await Library_depthupdate(dbHelper, this, this.path);
        await Library_doUpdate(dbHelper, this)
    } catch (err) {
        console.err(`Cannot update Library ${this.path}:`, err);
    }
    let timeDiff = (Date.now() - startTime) / 1000;
    const timeDiffMin = Math.floor(timeDiff / 60);
    timeDiff -= timeDiffMin*60;
    console.log(`Done updating library ${this.path}. Managed ${this.foundMedias.length} new media files. in ${timeDiffMin}m ${timeDiff}s`);
}

Library.prototype.findMedia = async function(p) {
    if (!p.startsWith(this.path))
        return null;
    let result = new File(p, path.basename(p));
    await result.enrich();
    return result;
}

module.exports.Library = Library;