storage.hpp

/*
 * LMDB Abstraction Layer.
 * Copyright (C) 2023  Yury Gubich <blue@macaw.me>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
#pragma once
 
#include "storage.h"
#include "exceptions.h"
 
#define UNUSED(x) (void)(x)
 
template<class K, class V>
LMDBAL::Storage<K, V>::Storage(Base* parent, const std::string& name, bool duplicates):
    iStorage(parent, name, duplicates),
    keySerializer(),
    valueSerializer(),
    cursors()
{}
 
template<class K, class V>
LMDBAL::Storage<K, V>::~Storage() {
    for (const std::pair<const uint32_t, Cursor<K, V>*>& pair : cursors)
        pair.second->dropped();
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::addRecord(const K& key, const V& value) {
    ensureOpened(addRecordMethodName);
    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::addRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    commitTransaction(txn);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::addRecord(const K& key, const V& value, TransactionID txn) {
    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData = valueSerializer.setData(value);
 
    unsigned int flags = 0;
    if (duplicates)
        flags |= MDB_NODUPDATA;
    else
        flags |= MDB_NOOVERWRITE;
 
    int rc = _mdbPut(txn, lmdbKey, lmdbData, flags);
    if (rc != MDB_SUCCESS)
        throwDuplicateOrUnknown(rc, toString(key));
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::addRecord(const K& key, const V& value, const WriteTransaction& txn) {
    ensureOpened(addRecordMethodName);
    addRecord(key, value, extractTransactionId(txn, addRecordMethodName));
}
 
template<class K, class V>
bool LMDBAL::Storage<K, V>::forceRecord(const K& key, const V& value) {
    ensureOpened(forceRecordMethodName);
 
    TransactionID txn = beginTransaction();
    bool added;
    try {
        added = Storage<K, V>::forceRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    commitTransaction(txn);
    return added;
}
 
template<class K, class V>
bool LMDBAL::Storage<K, V>::forceRecord(const K& key, const V& value, TransactionID txn) {
    bool added;
    if (duplicates) {
        try {
            addRecord(key, value, txn);
            added = true;
        } catch (const LMDBAL::Exist& e) {
            added = false;
        }
    } else {
        MDB_val lmdbKey = keySerializer.setData(key);
        MDB_val lmdbData;
 
        int rc = _mdbGet(txn, lmdbKey, lmdbData);
        switch (rc) {
            case MDB_SUCCESS:
                added = false;
                break;
            case MDB_NOTFOUND:
                added = true;
                break;
            default:
                added = false;
                throwUnknown(rc);
        }
 
        lmdbData = valueSerializer.setData(value);
        rc = _mdbPut(txn, lmdbKey, lmdbData);
        if (rc != MDB_SUCCESS)
            throwUnknown(rc);
    }
    return added;
}
 
template<class K, class V>
bool LMDBAL::Storage<K, V>::forceRecord(const K& key, const V& value, const WriteTransaction& txn) {
    ensureOpened(forceRecordMethodName);
    return forceRecord(key, value, extractTransactionId(txn, forceRecordMethodName));
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::changeRecord(const K& key, const V& value) {
    ensureOpened(changeRecordMethodName);
 
    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::changeRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    commitTransaction(txn);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::changeRecord(const K& key, const V& value, TransactionID txn) {
    MDB_cursor* cursor;
    int rc = _mdbCursorOpen(txn, &cursor);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);
 
    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData;
    rc = _mdbCursorGet(cursor, lmdbKey, lmdbData, MDB_SET);
    if (rc != MDB_SUCCESS)
        throwNotFoundOrUnknown(rc, toString(key));
 
    MDB_val lmdbNewData = valueSerializer.setData(value);
    bool sameSize = lmdbData.mv_size == lmdbNewData.mv_size;
    int firstDifferentByte = 0;
    if (sameSize) {                                             //can compare only if they are the same size
        firstDifferentByte = memcmp(lmdbData.mv_data, lmdbNewData.mv_data, lmdbData.mv_size);
        if (firstDifferentByte == 0) {                          //old and new is the same, nothing to do
            _mdbCursorClose(cursor);
            return;
        }
    }
 
    unsigned int flags = MDB_CURRENT;
    if (duplicates && (!sameSize || firstDifferentByte < 0)) {  //if new value is greater than the old one
        rc = _mdbCursorDel(cursor);                             //we need to initiate duplicates sort, for it to be in the correct place
        flags = MDB_NODUPDATA;
    }
 
    if (rc == MDB_SUCCESS)
        rc = _mdbCursorPut(cursor, lmdbKey, lmdbNewData, flags);
 
    _mdbCursorClose(cursor);
    if (rc != MDB_SUCCESS)
        throwDuplicateOrUnknown(rc, toString(key));
}
 
 
template<class K, class V>
void LMDBAL::Storage<K, V>::changeRecord(const K& key, const V& value, const WriteTransaction& txn) {
    ensureOpened(changeRecordMethodName);
    changeRecord(key, value, extractTransactionId(txn, changeRecordMethodName));
}
 
template<class K, class V>
V LMDBAL::Storage<K, V>::getRecord(const K& key) const {
    ensureOpened(getRecordMethodName);
 
    V value;
    Storage<K, V>::getRecord(key, value);
    return value;
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::getRecord(const K& key, V& value) const {
    ensureOpened(getRecordMethodName);
 
    TransactionID txn = beginReadOnlyTransaction();
    try {
        Storage<K, V>::getRecord(key, value, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    abortTransaction(txn);
}
 
template<class K, class V>
V LMDBAL::Storage<K, V>::getRecord(const K& key, TransactionID txn) const {
    V value;
    Storage<K, V>::getRecord(key, value, txn);
    return value;
}
 
template<class K, class V>
V LMDBAL::Storage<K, V>::getRecord(const K& key, const Transaction& txn) const {
    ensureOpened(getRecordMethodName);
    return getRecord(key, extractTransactionId(txn, getRecordMethodName));
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::getRecord(const K& key, V& value, TransactionID txn) const {
    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData;
 
    int rc = _mdbGet(txn, lmdbKey, lmdbData);
    if (rc != MDB_SUCCESS)
        throwNotFoundOrUnknown(rc, toString(key));
 
    valueSerializer.deserialize(lmdbData, value);
}
 
 
template<class K, class V>
void LMDBAL::Storage<K, V>::getRecord(const K& key, V& value, const Transaction& txn) const {
    ensureOpened(getRecordMethodName);
    getRecord(key, value, extractTransactionId(txn, getRecordMethodName));
}
 
template<class K, class V>
bool LMDBAL::Storage<K, V>::checkRecord(const K& key) const {
    ensureOpened(checkRecordMethodName);
 
    TransactionID txn = beginReadOnlyTransaction();
    bool result;
    try {
        result = Storage<K, V>::checkRecord(key, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    abortTransaction(txn);
    return result;
}
 
template<class K, class V>
bool LMDBAL::Storage<K, V>::checkRecord(const K& key, TransactionID txn) const {
    MDB_val lmdbKey = keySerializer.setData(key);
    MDB_val lmdbData;
 
    int rc = _mdbGet(txn, lmdbKey, lmdbData);
    if (rc == MDB_SUCCESS)
        return true;
 
    if (rc != MDB_NOTFOUND)
        throwUnknown(rc);
 
    return false;
}
 
template<class K, class V>
bool LMDBAL::Storage<K, V>::checkRecord(const K& key, const Transaction& txn) const {
    ensureOpened(checkRecordMethodName);
    return checkRecord(key, extractTransactionId(txn, checkRecordMethodName));
}
 
template<class K, class V>
std::map<K, V> LMDBAL::Storage<K, V>::readAll() const {
    ensureOpened(readAllMethodName);
 
    std::map<K, V> result;
    Storage<K, V>::readAll(result);
    return result;
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::readAll(std::map<K, V>& result) const {
    ensureOpened(readAllMethodName);
 
    TransactionID txn = beginReadOnlyTransaction();
    try {
        Storage<K, V>::readAll(result, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    abortTransaction(txn);
}
 
template<class K, class V>
std::map<K, V> LMDBAL::Storage<K, V>::readAll(TransactionID txn) const {
    std::map<K, V> result;
    Storage<K, V>::readAll(result, txn);
    return result;
}
 
template<class K, class V>
std::map<K, V> LMDBAL::Storage<K, V>::readAll(const Transaction& txn) const {
    ensureOpened(readAllMethodName);
    return readAll(extractTransactionId(txn, readAllMethodName));
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::readAll(std::map<K, V>& result, TransactionID txn) const {
    MDB_cursor* cursor;
    MDB_val lmdbKey, lmdbData;
 
    int rc = _mdbCursorOpen(txn, &cursor);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);
 
    rc = _mdbCursorGet(cursor, lmdbKey, lmdbData, MDB_FIRST);
    while (rc == MDB_SUCCESS) {
        K key;
        keySerializer.deserialize(lmdbKey, key);
        std::pair<typename std::map<K, V>::iterator, bool> probe = result.emplace(key, V{});
        if (probe.second)                                               //I do this to avoid overwrites in case duplicates are enabled
            valueSerializer.deserialize(lmdbData, probe.first->second);
 
        rc = _mdbCursorGet(cursor, lmdbKey, lmdbData, MDB_NEXT);
    }
    _mdbCursorClose(cursor);
    if (rc != MDB_NOTFOUND)
        throwUnknown(rc);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::readAll(std::map<K, V>& result, const Transaction& txn) const {
    ensureOpened(readAllMethodName);
    readAll(result, extractTransactionId(txn, readAllMethodName));
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::replaceAll(const std::map<K, V>& data) {
    ensureOpened(replaceAllMethodName);
 
    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::replaceAll(data, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    commitTransaction(txn);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::replaceAll(const std::map<K, V>& data, TransactionID txn) {
    int rc = drop(txn);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);
 
    MDB_val lmdbKey, lmdbData;
    for (const std::pair<const K, V>& pair : data) {
        lmdbKey = keySerializer.setData(pair.first);
        lmdbData = valueSerializer.setData(pair.second);
 
        rc = _mdbPut(txn, lmdbKey, lmdbData, MDB_NOOVERWRITE);   //TODO may be appending with cursor makes sence here?
        if (rc != MDB_SUCCESS)
            throwUnknown(rc);
    }
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::replaceAll(const std::map<K, V>& data, const WriteTransaction& txn) {
    ensureOpened(replaceAllMethodName);
    replaceAll(data, extractTransactionId(txn, replaceAllMethodName));
}
 
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::addRecords(const std::map<K, V>& data, bool overwrite) {
    ensureOpened(addRecordsMethodName);
 
    TransactionID txn = beginTransaction();
    uint32_t amount;
    try {
        amount = Storage<K, V>::addRecords(data, txn, overwrite);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    commitTransaction(txn);
    return amount;
}
 
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::addRecords(const std::map<K, V>& data, TransactionID txn, bool overwrite) {
    MDB_val lmdbKey, lmdbData;
    int rc;
    for (const std::pair<const K, V>& pair : data) {
        lmdbKey = keySerializer.setData(pair.first);
        lmdbData = valueSerializer.setData(pair.second);
 
        rc = _mdbPut(txn, lmdbKey, lmdbData, overwrite ? 0 : MDB_NOOVERWRITE);
        if (rc == MDB_KEYEXIST)
            throwDuplicate(toString(pair.first));
 
        if (rc != MDB_SUCCESS)
            throwUnknown(rc);
    }
 
    MDB_stat stat;
    rc = _mdbStat(txn, stat);
    if (rc != MDB_SUCCESS)
        throwUnknown(rc);
 
    return stat.ms_entries;
}
 
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::addRecords(const std::map<K, V>& data, const WriteTransaction& txn, bool overwrite) {
    ensureOpened(addRecordsMethodName);
    return addRecords(data, extractTransactionId(txn, addRecordsMethodName), overwrite);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::removeRecord(const K& key) {
    ensureOpened(removeRecordMethodName);
 
    TransactionID txn = beginTransaction();
    try {
        Storage<K, V>::removeRecord(key, txn);
    } catch (...) {
        abortTransaction(txn);
        throw;
    }
 
    commitTransaction(txn);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::removeRecord(const K& key, TransactionID txn) {
    MDB_val lmdbKey = keySerializer.setData(key);
    int rc = _mdbDel(txn, lmdbKey);
    if (rc != MDB_SUCCESS)
        throwNotFoundOrUnknown(rc, toString(key));
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::removeRecord(const K& key, const WriteTransaction& txn) {
    ensureOpened(removeRecordMethodName);
    removeRecord(key, extractTransactionId(txn, removeRecordMethodName));
}
 
template<class K, class V>
int LMDBAL::Storage<K, V>::open(MDB_txn* transaction) {
    return makeStorage<K, V>(transaction, duplicates);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::close() {
    for (const std::pair<const uint32_t, Cursor<K, V>*>& pair : cursors)
        pair.second->terminated();
 
    iStorage::close();
}
 
template<class K, class V>
LMDBAL::Cursor<K, V> LMDBAL::Storage<K, V>::createCursor() {
    return Cursor<K, V>(this);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::destroyCursor(LMDBAL::Cursor<K, V>& cursor) {
    typename std::map<uint32_t, Cursor<K, V>*>::iterator itr = cursors.find(cursor.id);
    if (itr == cursors.end())
        throwUnknown("An attempt to destroy a cursor the storage doesn't own");
 
    cursor.close();
    cursors.erase(itr);
    cursor.freed();
}
 
template<class K, class V>
uint32_t LMDBAL::Storage<K, V>::flags() const {
    ensureOpened(flagsMethodName);
    uint32_t result;
    TransactionID txn = beginReadOnlyTransaction();
 
    int res = _mdbFlags(txn, result);
    abortTransaction(txn);
    if (res != MDB_SUCCESS)
        throwUnknown(res);
 
    return result;
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::discoveredRecord(const K& key, const V& value) const {
    UNUSED(key);
    UNUSED(value);
}
 
template<class K, class V>
void LMDBAL::Storage<K, V>::discoveredRecord(const K& key, const V& value, TransactionID txn) const {
    UNUSED(key);
    UNUSED(value);
    UNUSED(txn);
}
 
template<class K, class V>
inline int LMDBAL::iStorage::makeStorage(MDB_txn* transaction, bool duplicates) {
    unsigned int flags = MDB_CREATE;
    if constexpr (std::is_integral<K>::value)
        flags |= MDB_INTEGERKEY;
 
    if (duplicates) {
        flags |= MDB_DUPSORT;
 
        if constexpr (std::is_scalar<V>::value)
            flags |= MDB_DUPFIXED;
 
        if constexpr (
            std::is_same<V, uint32_t>::value ||
            std::is_same<V, int32_t>::value ||
            std::is_same<V, uint64_t>::value ||
            std::is_same<V, int64_t>::value
        )                                   //for some reason lmdb breaks if it's not one of these types in MDB_DUPFIXED mode
            flags |= MDB_INTEGERDUP;
    }
 
    return _mdbOpen(transaction, flags);
}
 
template<class T>
inline std::string LMDBAL::iStorage::toString(const T& value) {
    return std::to_string(value);
}
 
template<>
inline std::string LMDBAL::iStorage::toString(const QString& value) {
    return value.toStdString();
}
 
template<>
inline std::string LMDBAL::iStorage::toString(const std::string& value) {
    return value;
}