electron-rare/FreeCAD
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

App: improve code quality of StringHasher

Browse Source
This commit is contained in:
Zheng, Lei
2022-10-01 04:49:07 +02:00
committed by Uwe
parent 6df100f5b7
commit 015d6396ff
2 changed files with 224 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/App/StringHasher.cpp
+28 -19
View File
@@ -150,7 +150,7 @@ void StringID::mark() const
{
if (isMarked())
return;
_flags.set(Marked);
_flags.setFlag(Flag::Marked);
for (auto & sid : _sids)
sid.deref().mark();
}
@@ -222,11 +222,15 @@ long StringHasher::lastID() const {
StringIDRef StringHasher::getID(const char *text, int len, bool hashable) {
if(len<0) len = strlen(text);
return getID(QByteArray::fromRawData(text,len),false,hashable);
return getID(QByteArray::fromRawData(text,len),hashable?Option::Hashable:Option::None);
}
StringIDRef StringHasher::getID(const QByteArray &data, bool binary, bool hashable, bool nocopy)
StringIDRef StringHasher::getID(const QByteArray &data, Options options)
{
bool binary = options.testFlag(Option::Binary);
bool hashable = options.testFlag(Option::Hashable);
bool nocopy = options.testFlag(Option::NoCopy);
bool hashed = hashable && _hashes->Threshold>0
&& (int)data.size()>_hashes->Threshold;
@@ -247,7 +251,12 @@ StringIDRef StringHasher::getID(const QByteArray &data, bool binary, bool hashab
d._data = QByteArray(data.constData(), data.size());
}
StringIDRef sid(new StringID(lastID()+1,d._data,binary,hashed));
StringID::Flags flags(StringID::Flag::None);
if (binary)
flags.setFlag(StringID::Flag::Binary);
if (hashed)
flags.setFlag(StringID::Flag::Hashed);
StringIDRef sid(new StringID(lastID()+1,d._data,flags));
return StringIDRef(insert(sid));
}
@@ -279,18 +288,18 @@ StringIDRef StringHasher::getID(const Data::MappedName &name,
StringIDRef postfixRef;
if (d._postfix.size() && d._postfix.indexOf("#") < 0) {
postfixRef = getID(d._postfix, false, false);
postfixRef = getID(d._postfix);
postfixRef.toBytes(d._postfix);
}
StringIDRef indexRef;
if (indexed)
indexRef = getID(d._data, false, false);
indexRef = getID(d._data);
StringIDRef sid(new StringID(lastID()+1,d._data,false,false));
StringIDRef sid(new StringID(lastID()+1,d._data));
StringID & id = *sid._sid;
if (d._postfix.size()) {
id._flags.set(StringID::Postfixed);
id._flags.setFlag(StringID::Flag::Postfixed);
id._postfix = d._postfix;
}
@@ -306,11 +315,11 @@ StringIDRef StringHasher::getID(const Data::MappedName &name,
else {
id._sids.reserve(count + extra);
if (postfixRef) {
id._flags.set(StringID::PostfixEncoded);
id._flags.setFlag(StringID::Flag::PostfixEncoded);
id._sids.push_back(postfixRef);
}
if (indexRef) {
id._flags.set(StringID::Indexed);
id._flags.setFlag(StringID::Flag::Indexed);
id._sids.push_back(indexRef);
}
for (auto &s : sids) {
@@ -332,9 +341,9 @@ StringIDRef StringHasher::getID(const Data::MappedName &name,
if (i!=offset)
std::swap(id._sids[offset], id._sids[i]);
if (res.index != 0)
id._flags.set(StringID::PrefixIDIndex);
id._flags.setFlag(StringID::Flag::PrefixIDIndex);
else
id._flags.set(StringID::PrefixID);
id._flags.setFlag(StringID::Flag::PrefixID);
break;
}
}
@@ -457,8 +466,8 @@ void StringHasher::saveStream(std::ostream &s) const {
}
auto flags = d._flags;
flags.reset(StringID::Marked);
s << '.' << flags.to_ulong();
flags.setFlag(StringID::Flag::Marked, false);
s << '.' << flags.toUnderlyingType();
int i = 0;
if (!relative) {
@@ -551,7 +560,7 @@ void StringHasher::restoreStreamNew(std::istream &s, std::size_t count) {
lastid = id;
unsigned long flag = strtol(tokens[1].c_str(), nullptr, 16);
StringIDRef sid(new StringID(id,QByteArray(),flag));
StringIDRef sid(new StringID(id,QByteArray(),static_cast<StringID::Flag>(flag)));
StringID & d = *sid._sid;
d._sids.reserve(tokens.size()-2);
@@ -644,7 +653,7 @@ void StringHasher::restoreStream(std::istream &s, std::size_t count) {
int32_t id;
uint8_t type;
str >> id >> type >> content;
StringIDRef sid = new StringID(id,QByteArray(),type);
StringIDRef sid = new StringID(id,QByteArray(),static_cast<StringID::Flag>(type));
if(sid.isHashed() || sid.isBinary()) {
sid._sid->_data = QByteArray::fromBase64(content.c_str());
} else
@@ -708,9 +717,9 @@ void StringHasher::Restore(Base::XMLReader &reader) {
bool hashed = reader.hasAttribute("hash");
if(hashed || reader.hasAttribute("data")) {
const char* value = hashed?reader.getAttribute("hash"):reader.getAttribute("data");
sid = new StringID(id,QByteArray::fromBase64(value),true,hashed);
sid = new StringID(id,QByteArray::fromBase64(value),StringID::Flag::Hashed);
}else {
sid = new StringID(id,QByteArray(reader.getAttribute("text")),false,false);
sid = new StringID(id,QByteArray(reader.getAttribute("text")));
}
insert(sid);
}
@@ -736,5 +745,5 @@ std::map<long,StringIDRef> StringHasher::getIDMap() const {
void StringHasher::clearMarks() const
{
for (auto & v : _hashes->right)
v.second->_flags.reset(StringID::Marked);
v.second->_flags.setFlag(StringID::Flag::Marked, false);
}
src/App/StringHasher.h
+196 -35
View File
@@ -32,6 +32,7 @@
#include <CXX/Objects.hxx>
#include <Base/Handle.h>
#include <Base/Persistence.h>
#include <Base/Bitmask.h>
namespace Data{
class MappedName;
@@ -44,68 +45,128 @@ class StringID;
class StringIDRef;
typedef Base::Reference<StringHasher> StringHasherRef;
/** Class to store a string
*
* The main purpose of this class is to provide an efficient storage of the
* mapped geometry element name (i.e. the new Topological Naming), but it can
* also be used as a general purpose string table.
*
* The StringID is to be stored in a string table (StringHasher), and be
* referred to by an integer ID. The stored data can be optionally divided into
* two parts, prefix and postfix. This is because a new mapped name is often
* created by adding some common postfix to an existing name, so data sharing
* can be improved using the following techniques:
*
* a) reference count (through QByteArray) the main data part,
*
* b) (recursively) encode prefix and/or postfix as an integer (in the
* format of #<hex>, e.g. #1b) that references another StringID,
*
* c) Check index based name in prefix, e.g. Edge1, Vertex2, and encode
* only the text part as StringID. The index is stored separately in
* reference class StringIDRef to maximize data sharing.
*/
class AppExport StringID: public Base::BaseClass, public Base::Handled {
TYPESYSTEM_HEADER_WITH_OVERRIDE();
public:
enum Flag {
Binary,
Hashed,
PostfixEncoded,
Postfixed,
Indexed,
PrefixID,
PrefixIDIndex,
Persistent,
Marked,
/// Flag of the stored string data
enum class Flag {
/// No flag
None = 0,
/// The stored data is binary
Binary = 1 << 0,
/// The stored data is the sha1 hash of the original content
Hashed = 1 << 1,
/** Postfix is encoded as #<hex>, e.g. #1b, where the hex integer part
* refers to another StringID.
*/
PostfixEncoded = 1 << 2,
/// The data is splited as prefix and postfix
Postfixed = 1 << 3,
/// The prefix data is split as text + index
Indexed = 1 << 4,
/** The prefix data is encoded as #<hex>, e.g. #1b, where the hex
* integer part refers to another StringID.
*/
PrefixID = 1 << 5,
/** The prefix split as text + index, where the text is encoded
* using another StringID.
*/
PrefixIDIndex = 1 << 6,
/// The string ID is persistent regardless of internal mark */
Persistent = 1 << 7,
/// Internal marked used to check if the string ID is used
Marked = 1 << 8,
};
StringID(long id, const QByteArray &data, bool binary, bool hashed)
:_id(id),_data(data)
{
if(binary) _flags.set(Binary);
if(hashed) _flags.set(Hashed);
}
typedef Base::Flags<Flag> Flags;
StringID(long id, const QByteArray &data, uint8_t flags)
/** Constructor
* @param id: integer ID of this StringID
* @param data: input data
* @param flags: flags describes the data
*
* User code is not supposed to create StringID directly, but through StringHasher::getID()
*/
StringID(long id, const QByteArray &data, const Flags &flags=Flag::None)
:_id(id),_data(data),_flags(flags)
{}
/// Constructs an empty StringID
StringID()
:_id(0),_flags(0)
:_id(0), _flags(Flag::None)
{}
virtual ~StringID();
/// Returns the ID of this StringID
long value() const {return _id;}
/// Returns all related StringIDs that used to encode this StringID
const QVector<StringIDRef> &relatedIDs() const {return _sids;}
bool isBinary() const {return _flags.test(Binary);}
bool isHashed() const {return _flags.test(Hashed);}
bool isPostfixed() const {return _flags.test(Postfixed);}
bool isPostfixEncoded() const {return _flags.test(PostfixEncoded);}
bool isIndexed() const {return _flags.test(Indexed);}
bool isPrefixID() const {return _flags.test(PrefixID);}
bool isPrefixIDIndex() const {return _flags.test(PrefixIDIndex);}
bool isMarked() const {return _flags.test(Marked);}
bool isPersistent() const {return _flags.test(Persistent);}
/// @name Flag accessors
//@{
bool isBinary() const {return _flags.testFlag(Flag::Binary);}
bool isHashed() const {return _flags.testFlag(Flag::Hashed);}
bool isPostfixed() const {return _flags.testFlag(Flag::Postfixed);}
bool isPostfixEncoded() const {return _flags.testFlag(Flag::PostfixEncoded);}
bool isIndexed() const {return _flags.testFlag(Flag::Indexed);}
bool isPrefixID() const {return _flags.testFlag(Flag::PrefixID);}
bool isPrefixIDIndex() const {return _flags.testFlag(Flag::PrefixIDIndex);}
bool isMarked() const {return _flags.testFlag(Flag::Marked);}
bool isPersistent() const {return _flags.testFlag(Flag::Persistent);}
//@}
/// Checks if this StringID is from the input hasher
bool isFromSameHasher(const StringHasherRef & hasher) const
{
return this->_hasher == hasher;
}
/// Returns the owner hasher
StringHasherRef getHasher() const
{
return StringHasherRef(_hasher);
}
/// Returns the data (prefix)
const QByteArray data() const {return _data;}
/// Returns the postfix
const QByteArray postfix() const {return _postfix;}
virtual PyObject *getPyObject() override;
/// Returns a Python tuple containing both the text and index
PyObject *getPyObjectWithIndex(int index);
/** Convert to string represtation of this StringID
* @param index: optional index
*
* The format is #<id>. And if index is non zero, then #<id>:<index>. Both
* <id> and <index> are in hex format.
*/
std::string toString(int index) const;
/// Light weight structure of holding a string ID and associated index
struct IndexID {
long id;
int index;
@@ -121,14 +182,44 @@ public:
return s;
}
};
/** Parse string to get ID and index
* @param name: input string
* @param eof: Whether to check the end of string. If true, then the input
* string must contain only the string representation of this
* StringID
* @param size: input string size, or -1 if the input string is zero terminated.
* @return Return the integer ID and index.
*
* The input string is expected to be in the format of #<id> or with index
* #<id>:<index>, where both id and index are in hex digits.
*/
static IndexID fromString(const char *name, bool eof=true, int size = -1);
/** Parse string to get ID and index
* @param bytes: input data
* @param eof: Whether to check the end of string. If true, then the input
* string must contain only the string representation of this
* StringID
*
* The input string is expected to be in the format of #<id> or with index
* #<id>:<index>, where both id and index are in hex digits.
*/
static IndexID fromString(const QByteArray &bytes, bool eof=true) {
return fromString(bytes.constData(), eof, bytes.size());
}
/** Get the text content of this StringID
* @param index: optional index
* @return Return the text content of this StringID. If the data is binary,
* then output in base64 encoded string.
*/
std::string dataToText(int index) const;
/** Get the content of this StringID as QByteArray
* @param bytes: output bytes
* @param index: opttional index.
*/
void toBytes(QByteArray &bytes, int index) const {
if (_postfix.size())
bytes = _data + _postfix;
@@ -138,17 +229,23 @@ public:
bytes = _data;
}
/// Mark this StringID as used
void mark() const;
/// Mark the StringID as persistent regardless of usage mark
void setPersistent(bool enable)
{
_flags.set(Persistent, enable);
_flags.setFlag(Flag::Persistent, enable);
}
bool operator<(const StringID &other) const {
return compare(other) < 0;
}
/** Compare StringID
* @param other: the other StringID for comparison
* @return Returns -1 if less than the other StringID, 1 if greater, or 0 if equal
*/
int compare(const StringID &other) const {
if (_hasher < other._hasher)
return -1;
@@ -168,17 +265,19 @@ private:
QByteArray _data;
QByteArray _postfix;
StringHasher *_hasher = nullptr;
mutable std::bitset<32> _flags;
mutable Flags _flags;
mutable QVector<StringIDRef> _sids;
};
//////////////////////////////////////////////////////////////////////////
/** Counted reference to a StringID instance
*/
class StringIDRef
{
public:
StringIDRef()
:_sid(0), _index(0)
:_sid(nullptr), _index(0)
{}
StringIDRef(StringID* p, int index=0)
@@ -413,28 +512,67 @@ public:
const std::string &getPersistenceFileName() const;
/** Maps an arbitrary string to an integer
*
* @param text: input string.
* @param len: length of the string, or -1 if the string is 0 terminated.
* @param hashable: whether the string is hashable.
* @return Return a shared pointer to the internally stored StringID.
*
* The function maps an arbitrary text string to a unique integer ID, which
* is returned as a shared pointer to reference count the ID so that it is
* possible to prune any unused strings.
*
* If the string is longer than the threshold setting of this StringHasher,
* it will be sha1 hashed before storing, and the original content of the
* string is discarded.
* If \c hashable is true and the string is longer than the threshold
* setting of this StringHasher, it will be sha1 hashed before storing, and
* the original content of the string is discarded. If else, the string is
* copied and stored inside a StringID instance.
*
* The purpose of function is to provide a short form of a stable string
* identification.
*/
StringIDRef getID(const char *text, int len=-1, bool hashable=false);
/** Map text or binary data to an integer */
StringIDRef getID(const QByteArray & data, bool binary, bool hashable=true, bool nocopy=false);
/// Option for string string data
enum class Option {
/// No option
None = 0,
/// The input data is binary
Binary = 1 << 0,
/** The input data is hashable. If the data length is longer than the
* threshold setting of the StringHasher, it will be sha1 hashed before
* storing, and the original content of the string is discarded.
*/
Hashable = 1 << 1,
/// Do not copy the data, assuming the data is constant. If this option
//is not set, the data will be copied before storing.
NoCopy = 1 << 2,
};
typedef Base::Flags<Option> Options;
/** Map text or binary data to an integer
*
* @param data: input data.
* @param options: options describing how to store the data. @sa Option.
* @return Return a shared pointer to the internally stored StringID.
*
* The function maps an arbitrary text string to a unique integer ID, which
* is returned as a shared pointer to reference count the ID so that it is
* possible to prune any unused strings.
*
* The purpose of function is to provide a short form of a stable string
* identification.
*/
StringIDRef getID(const QByteArray & data, Options options=Option::Hashable);
/** Map geometry element name to an integer */
StringIDRef getID(const Data::MappedName & name,
const QVector<StringIDRef> & sids);
/** Obtain the reference counted StringID object from numerical id
*
* @param id: string ID
* @param index: optional index of the string ID
* @return Return a shared pointer to the internally stored StringID.
*
* This function exists because the stored string may be one way hashed,
* and the original text is not persistent. The caller use this function to
@@ -442,6 +580,11 @@ public:
*/
StringIDRef getID(long id, int index = 0) const;
/** Obtain the reference counted StringID object from numerical id and index
*
* @param id: string ID with index
* @return Return a shared pointer to the internally stored StringID.
*/
StringIDRef getID(const StringID::IndexID &id) const {
return getID(id.id, id.index);
}
@@ -459,14 +602,29 @@ public:
virtual PyObject *getPyObject(void) override;
/** Enable/disable saving all string ID
*
* If disabled, then only save string ID that are used.
*/
void setSaveAll(bool enable);
bool getSaveAll() const;
/** Set threshold of string hashing
*
* For hashable string that are longer than the threshold, the string will
* be replaced by its sha1 hash.
*/
void setThreshold(int threshold);
int getThreshold() const;
/** Clear internal marks
*
* The internal marks on internally stored StringID instances are used to
* check if the StringID is used.
*/
void clearMarks() const;
/// Compact string storage
void compact();
class HashMap;
@@ -486,4 +644,7 @@ private:
}
ENABLE_BITMASK_OPERATORS(App::StringID::Flag)
ENABLE_BITMASK_OPERATORS(App::StringHasher::Option)
#endif